S2S_TRAIN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "S2S-TRAIN.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "julia-1.2",
      "display_name": "Julia 1.0"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/lanxaustralis/NLP-Projects/blob/master/S2S_TRAIN.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oMSuTc3pDlHv",
        "colab_type": "code",
        "outputId": "8dee89c5-9acc-49c2-9c65-f44404f1d608",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!wget https://developer.nvidia.com/compute/cuda/9.0/Prod/local_installers/cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb\n",
        "!dpkg -i cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb\n",
        "!apt-key add /var/cuda-repo-9-0-local/7fa2af80.pub\n",
        "!apt update -q\n",
        "!apt install cuda gcc-6 g++-6 -y -q\n",
        "!ln -s /usr/bin/gcc-6 /usr/local/cuda/bin/gcc\n",
        "!ln -s /usr/bin/g++-6 /usr/local/cuda/bin/g++\n",
        "!curl -sSL \"https://julialang-s3.julialang.org/bin/linux/x64/1.2/julia-1.2.0-linux-x86_64.tar.gz\" -o julia.tar.gz\n",
        "!tar -xzf julia.tar.gz -C /usr --strip-components 1\n",
        "!rm -rf julia.tar.gz*\n",
        "!julia -e 'using Pkg; pkg\"add IJulia; add Knet; precompile\"'\n",
        "# Restart with new IPython.\n",
        "import os\n",
        "import signal\n",
        "os.kill(os.getpid(), signal.SIGTERM)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-11-04 19:01:40--  https://developer.nvidia.com/compute/cuda/9.0/Prod/local_installers/cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb\n",
            "Resolving developer.nvidia.com (developer.nvidia.com)... 192.229.182.215\n",
            "Connecting to developer.nvidia.com (developer.nvidia.com)|192.229.182.215|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://developer.download.nvidia.com/compute/cuda/9.0/secure/Prod/local_installers/cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64.deb?T2taPXUGfsB-9whnBSrXdsTP8yjb_SuMfh-PDCMH8V0QEWOYRRl4tDIByTtKerrKsFJPCBCLv7z_4mcgezfxz6QQo3G0uAXtw4JzsGbIuNtGrd5ku76eFLs76Giq-DyPKSDD3XJWTsVyUxGnREJtHBvMnx9XlWV5uuWWKC90b08aImcr-1WnBo9b67SHGIfSn8cNuklYC4_yD6qtxRU0 [following]\n",
            "--2019-11-04 19:01:41--  https://developer.download.nvidia.com/compute/cuda/9.0/secure/Prod/local_installers/cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64.deb?T2taPXUGfsB-9whnBSrXdsTP8yjb_SuMfh-PDCMH8V0QEWOYRRl4tDIByTtKerrKsFJPCBCLv7z_4mcgezfxz6QQo3G0uAXtw4JzsGbIuNtGrd5ku76eFLs76Giq-DyPKSDD3XJWTsVyUxGnREJtHBvMnx9XlWV5uuWWKC90b08aImcr-1WnBo9b67SHGIfSn8cNuklYC4_yD6qtxRU0\n",
            "Resolving developer.download.nvidia.com (developer.download.nvidia.com)... 192.229.221.58, 2606:2800:233:ef6:15dd:1ece:1d50:1e1\n",
            "Connecting to developer.download.nvidia.com (developer.download.nvidia.com)|192.229.221.58|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1212738714 (1.1G) [application/x-deb]\n",
            "Saving to: ‘cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb’\n",
            "\n",
            "cuda-repo-ubuntu160 100%[===================>]   1.13G   199MB/s    in 6.0s    \n",
            "\n",
            "2019-11-04 19:01:47 (192 MB/s) - ‘cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb’ saved [1212738714/1212738714]\n",
            "\n",
            "Selecting previously unselected package cuda-repo-ubuntu1604-9-0-local.\n",
            "(Reading database ... 132681 files and directories currently installed.)\n",
            "Preparing to unpack cuda-repo-ubuntu1604-9-0-local_9.0.176-1_amd64-deb ...\n",
            "Unpacking cuda-repo-ubuntu1604-9-0-local (9.0.176-1) ...\n",
            "Setting up cuda-repo-ubuntu1604-9-0-local (9.0.176-1) ...\n",
            "OK\n",
            "Get:1 file:/var/cuda-repo-9-0-local  InRelease\n",
            "Ign:1 file:/var/cuda-repo-9-0-local  InRelease\n",
            "Get:2 file:/var/cuda-repo-9-0-local  Release [574 B]\n",
            "Get:2 file:/var/cuda-repo-9-0-local  Release [574 B]\n",
            "Get:3 file:/var/cuda-repo-9-0-local  Release.gpg [819 B]\n",
            "Get:3 file:/var/cuda-repo-9-0-local  Release.gpg [819 B]\n",
            "Hit:4 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Hit:5 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:6 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "Get:7 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic InRelease [15.4 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Get:9 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "Get:10 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/ InRelease [3,626 B]\n",
            "Ign:11 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Get:12 file:/var/cuda-repo-9-0-local  Packages [15.4 kB]\n",
            "Ign:13 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:14 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:15 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Get:16 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic/main Sources [1,718 kB]\n",
            "Get:17 http://ppa.launchpad.net/marutter/c2d4u3.5/ubuntu bionic/main amd64 Packages [827 kB]\n",
            "Get:18 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [1,304 kB]\n",
            "Get:19 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [995 kB]\n",
            "Get:20 http://archive.ubuntu.com/ubuntu bionic-backports/universe amd64 Packages [4,234 B]\n",
            "Get:21 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/ Packages [73.5 kB]\n",
            "Get:24 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [782 kB]\n",
            "Get:25 http://security.ubuntu.com/ubuntu bionic-security/main amd64 Packages [700 kB]\n",
            "Fetched 6,676 kB in 3s (1,974 kB/s)\n",
            "Reading package lists...\n",
            "Building dependency tree...\n",
            "Reading state information...\n",
            "39 packages can be upgraded. Run 'apt list --upgradable' to see them.\n",
            "Reading package lists...\n",
            "Building dependency tree...\n",
            "Reading state information...\n",
            "The following additional packages will be installed:\n",
            "  cpp-6 cuda-10-1 cuda-command-line-tools-10-1 cuda-compiler-10-1\n",
            "  cuda-cudart-10-1 cuda-cudart-dev-10-1 cuda-cufft-10-1 cuda-cufft-dev-10-1\n",
            "  cuda-cuobjdump-10-1 cuda-cupti-10-1 cuda-curand-10-1 cuda-curand-dev-10-1\n",
            "  cuda-cusolver-10-1 cuda-cusolver-dev-10-1 cuda-cusparse-10-1\n",
            "  cuda-cusparse-dev-10-1 cuda-demo-suite-10-1 cuda-documentation-10-1\n",
            "  cuda-driver-dev-10-1 cuda-gdb-10-1 cuda-gpu-library-advisor-10-1\n",
            "  cuda-libraries-10-1 cuda-libraries-dev-10-1 cuda-license-10-1\n",
            "  cuda-memcheck-10-1 cuda-misc-headers-10-1 cuda-npp-10-1 cuda-npp-dev-10-1\n",
            "  cuda-nsight-10-1 cuda-nsight-compute-10-1 cuda-nsight-systems-10-1\n",
            "  cuda-nvcc-10-1 cuda-nvdisasm-10-1 cuda-nvgraph-10-1 cuda-nvgraph-dev-10-1\n",
            "  cuda-nvjpeg-10-1 cuda-nvjpeg-dev-10-1 cuda-nvml-dev-10-1 cuda-nvprof-10-1\n",
            "  cuda-nvprune-10-1 cuda-nvrtc-10-1 cuda-nvrtc-dev-10-1 cuda-nvtx-10-1\n",
            "  cuda-nvvp-10-1 cuda-runtime-10-1 cuda-samples-10-1 cuda-sanitizer-api-10-1\n",
            "  cuda-toolkit-10-1 cuda-tools-10-1 cuda-visual-tools-10-1 gcc-6-base libasan3\n",
            "  libcublas-dev libcublas10 libgcc-6-dev libstdc++-6-dev\n",
            "  nsight-compute-2019.4.0 nsight-systems-2019.3.7\n",
            "Suggested packages:\n",
            "  gcc-6-locales g++-6-multilib gcc-6-doc libstdc++6-6-dbg gcc-6-multilib\n",
            "  libgcc1-dbg libgomp1-dbg libitm1-dbg libatomic1-dbg libasan3-dbg\n",
            "  liblsan0-dbg libtsan0-dbg libubsan0-dbg libcilkrts5-dbg libmpx2-dbg\n",
            "  libquadmath0-dbg libstdc++-6-doc\n",
            "The following NEW packages will be installed:\n",
            "  cpp-6 cuda cuda-10-1 cuda-command-line-tools-10-1 cuda-compiler-10-1\n",
            "  cuda-cudart-10-1 cuda-cudart-dev-10-1 cuda-cufft-10-1 cuda-cufft-dev-10-1\n",
            "  cuda-cuobjdump-10-1 cuda-cupti-10-1 cuda-curand-10-1 cuda-curand-dev-10-1\n",
            "  cuda-cusolver-10-1 cuda-cusolver-dev-10-1 cuda-cusparse-10-1\n",
            "  cuda-cusparse-dev-10-1 cuda-demo-suite-10-1 cuda-documentation-10-1\n",
            "  cuda-driver-dev-10-1 cuda-gdb-10-1 cuda-gpu-library-advisor-10-1\n",
            "  cuda-libraries-10-1 cuda-libraries-dev-10-1 cuda-license-10-1\n",
            "  cuda-memcheck-10-1 cuda-misc-headers-10-1 cuda-npp-10-1 cuda-npp-dev-10-1\n",
            "  cuda-nsight-10-1 cuda-nsight-compute-10-1 cuda-nsight-systems-10-1\n",
            "  cuda-nvcc-10-1 cuda-nvdisasm-10-1 cuda-nvgraph-10-1 cuda-nvgraph-dev-10-1\n",
            "  cuda-nvjpeg-10-1 cuda-nvjpeg-dev-10-1 cuda-nvml-dev-10-1 cuda-nvprof-10-1\n",
            "  cuda-nvprune-10-1 cuda-nvrtc-10-1 cuda-nvrtc-dev-10-1 cuda-nvtx-10-1\n",
            "  cuda-nvvp-10-1 cuda-runtime-10-1 cuda-samples-10-1 cuda-sanitizer-api-10-1\n",
            "  cuda-toolkit-10-1 cuda-tools-10-1 cuda-visual-tools-10-1 g++-6 gcc-6\n",
            "  gcc-6-base libasan3 libcublas-dev libcublas10 libgcc-6-dev libstdc++-6-dev\n",
            "  nsight-compute-2019.4.0 nsight-systems-2019.3.7\n",
            "0 upgraded, 61 newly installed, 0 to remove and 39 not upgraded.\n",
            "Need to get 1,777 MB of archives.\n",
            "After this operation, 4,061 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 gcc-6-base amd64 6.5.0-2ubuntu1~18.04 [16.7 kB]\n",
            "Get:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-license-10-1 10.1.243-1 [16.4 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 cpp-6 amd64 6.5.0-2ubuntu1~18.04 [6,396 kB]\n",
            "Get:4 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-misc-headers-10-1 10.1.243-1 [692 kB]\n",
            "Get:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvcc-10-1 10.1.243-1 [15.7 MB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 libasan3 amd64 6.5.0-2ubuntu1~18.04 [313 kB]\n",
            "Get:7 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 libgcc-6-dev amd64 6.5.0-2ubuntu1~18.04 [2,308 kB]\n",
            "Get:8 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 gcc-6 amd64 6.5.0-2ubuntu1~18.04 [7,067 kB]\n",
            "Get:9 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-cuobjdump-10-1 10.1.243-1 [81.6 kB]\n",
            "Get:10 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvprune-10-1 10.1.243-1 [36.9 kB]\n",
            "Get:11 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-compiler-10-1 10.1.243-1 [2,534 B]\n",
            "Get:12 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvdisasm-10-1 10.1.243-1 [22.1 MB]\n",
            "Get:13 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 libstdc++-6-dev amd64 6.5.0-2ubuntu1~18.04 [1,437 kB]\n",
            "Get:14 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 g++-6 amd64 6.5.0-2ubuntu1~18.04 [7,213 kB]\n",
            "Get:15 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-gdb-10-1 10.1.243-1 [2,773 kB]\n",
            "Get:16 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvprof-10-1 10.1.243-1 [4,016 kB]\n",
            "Get:17 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-sanitizer-api-10-1 10.1.243-1 [2,153 kB]\n",
            "Get:18 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-memcheck-10-1 10.1.243-1 [139 kB]\n",
            "Get:19 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-cudart-10-1 10.1.243-1 [109 kB]\n",
            "Get:20 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-driver-dev-10-1 10.1.243-1 [11.6 kB]\n",
            "Get:21 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-cudart-dev-10-1 10.1.243-1 [447 kB]\n",
            "Get:22 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-cupti-10-1 10.1.243-1 [11.2 MB]\n",
            "Get:23 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-gpu-library-advisor-10-1 10.1.243-1 [1,003 kB]\n",
            "Get:24 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvtx-10-1 10.1.243-1 [38.9 kB]\n",
            "Get:25 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-command-line-tools-10-1 10.1.243-1 [27.3 kB]\n",
            "Get:26 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nsight-10-1 10.1.243-1 [2,584 B]\n",
            "Get:27 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvvp-10-1 10.1.243-1 [2,530 B]\n",
            "Get:28 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvrtc-10-1 10.1.243-1 [6,288 kB]\n",
            "Get:29 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvrtc-dev-10-1 10.1.243-1 [8,812 B]\n",
            "Get:30 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-cusolver-10-1 10.1.243-1 [71.9 MB]\n",
            "Get:31 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-cusolver-dev-10-1 10.1.243-1 [14.0 MB]\n",
            "Get:32 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  libcublas10 10.2.1.243-1 [38.8 MB]\n",
            "Get:33 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  libcublas-dev 10.2.1.243-1 [38.9 MB]\n",
            "Get:34 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-cufft-10-1 10.1.243-1 [84.2 MB]\n",
            "Get:35 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-cufft-dev-10-1 10.1.243-1 [158 MB]\n",
            "Get:36 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-curand-10-1 10.1.243-1 [38.9 MB]\n",
            "Get:37 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-curand-dev-10-1 10.1.243-1 [58.1 MB]\n",
            "Get:38 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-cusparse-10-1 10.1.243-1 [55.2 MB]\n",
            "Get:39 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-cusparse-dev-10-1 10.1.243-1 [55.7 MB]\n",
            "Get:40 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-npp-10-1 10.1.243-1 [54.9 MB]\n",
            "Get:41 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-npp-dev-10-1 10.1.243-1 [55.7 MB]\n",
            "Get:42 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvml-dev-10-1 10.1.243-1 [53.1 kB]\n",
            "Get:43 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvjpeg-10-1 10.1.243-1 [1,213 kB]\n",
            "Get:44 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvjpeg-dev-10-1 10.1.243-1 [1,153 kB]\n",
            "Get:45 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  nsight-compute-2019.4.0 2019.4.0.12-1 [254 MB]\n",
            "Get:46 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nsight-compute-10-1 10.1.243-1 [3,698 B]\n",
            "Get:47 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  nsight-systems-2019.3.7 2019.3.7.5-3837e03 [109 MB]\n",
            "Get:48 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nsight-systems-10-1 10.1.243-1 [3,130 B]\n",
            "Get:49 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvgraph-10-1 10.1.243-1 [41.7 MB]\n",
            "Get:50 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-nvgraph-dev-10-1 10.1.243-1 [34.2 MB]\n",
            "Get:51 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-visual-tools-10-1 10.1.243-1 [398 MB]\n",
            "Get:52 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-tools-10-1 10.1.243-1 [2,496 B]\n",
            "Get:53 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-samples-10-1 10.1.243-1 [64.5 MB]\n",
            "Get:54 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-documentation-10-1 10.1.243-1 [53.5 MB]\n",
            "Get:55 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-libraries-dev-10-1 10.1.243-1 [2,612 B]\n",
            "Get:56 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-toolkit-10-1 10.1.243-1 [2,828 B]\n",
            "Get:57 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-libraries-10-1 10.1.243-1 [2,588 B]\n",
            "Get:58 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-runtime-10-1 10.1.243-1 [2,532 B]\n",
            "Get:59 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-demo-suite-10-1 10.1.243-1 [3,851 kB]\n",
            "Get:60 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda-10-1 10.1.243-1 [2,560 B]\n",
            "Get:61 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  cuda 10.1.243-1 [2,510 B]\n",
            "Fetched 1,777 MB in 31s (56.7 MB/s)\n",
            "Extracting templates from packages: 100%\n",
            "Selecting previously unselected package gcc-6-base:amd64.\n",
            "(Reading database ... 132740 files and directories currently installed.)\n",
            "Preparing to unpack .../00-gcc-6-base_6.5.0-2ubuntu1~18.04_amd64.deb ...\n",
            "Unpacking gcc-6-base:amd64 (6.5.0-2ubuntu1~18.04) ...\n",
            "Selecting previously unselected package cpp-6.\n",
            "Preparing to unpack .../01-cpp-6_6.5.0-2ubuntu1~18.04_amd64.deb ...\n",
            "Unpacking cpp-6 (6.5.0-2ubuntu1~18.04) ...\n",
            "Selecting previously unselected package cuda-license-10-1.\n",
            "Preparing to unpack .../02-cuda-license-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-license-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-misc-headers-10-1.\n",
            "Preparing to unpack .../03-cuda-misc-headers-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-misc-headers-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvcc-10-1.\n",
            "Preparing to unpack .../04-cuda-nvcc-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvcc-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-cuobjdump-10-1.\n",
            "Preparing to unpack .../05-cuda-cuobjdump-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-cuobjdump-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvprune-10-1.\n",
            "Preparing to unpack .../06-cuda-nvprune-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvprune-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-compiler-10-1.\n",
            "Preparing to unpack .../07-cuda-compiler-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-compiler-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvdisasm-10-1.\n",
            "Preparing to unpack .../08-cuda-nvdisasm-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvdisasm-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-gdb-10-1.\n",
            "Preparing to unpack .../09-cuda-gdb-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-gdb-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvprof-10-1.\n",
            "Preparing to unpack .../10-cuda-nvprof-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvprof-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-sanitizer-api-10-1.\n",
            "Preparing to unpack .../11-cuda-sanitizer-api-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-sanitizer-api-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-memcheck-10-1.\n",
            "Preparing to unpack .../12-cuda-memcheck-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-memcheck-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-cudart-10-1.\n",
            "Preparing to unpack .../13-cuda-cudart-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-cudart-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-driver-dev-10-1.\n",
            "Preparing to unpack .../14-cuda-driver-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-driver-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-cudart-dev-10-1.\n",
            "Preparing to unpack .../15-cuda-cudart-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-cudart-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-cupti-10-1.\n",
            "Preparing to unpack .../16-cuda-cupti-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-cupti-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-gpu-library-advisor-10-1.\n",
            "Preparing to unpack .../17-cuda-gpu-library-advisor-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-gpu-library-advisor-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvtx-10-1.\n",
            "Preparing to unpack .../18-cuda-nvtx-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvtx-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-command-line-tools-10-1.\n",
            "Preparing to unpack .../19-cuda-command-line-tools-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-command-line-tools-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nsight-10-1.\n",
            "Preparing to unpack .../20-cuda-nsight-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nsight-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvvp-10-1.\n",
            "Preparing to unpack .../21-cuda-nvvp-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvvp-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvrtc-10-1.\n",
            "Preparing to unpack .../22-cuda-nvrtc-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvrtc-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvrtc-dev-10-1.\n",
            "Preparing to unpack .../23-cuda-nvrtc-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvrtc-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-cusolver-10-1.\n",
            "Preparing to unpack .../24-cuda-cusolver-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-cusolver-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-cusolver-dev-10-1.\n",
            "Preparing to unpack .../25-cuda-cusolver-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-cusolver-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package libcublas10.\n",
            "Preparing to unpack .../26-libcublas10_10.2.1.243-1_amd64.deb ...\n",
            "Unpacking libcublas10 (10.2.1.243-1) ...\n",
            "Selecting previously unselected package libcublas-dev.\n",
            "Preparing to unpack .../27-libcublas-dev_10.2.1.243-1_amd64.deb ...\n",
            "Unpacking libcublas-dev (10.2.1.243-1) ...\n",
            "Selecting previously unselected package cuda-cufft-10-1.\n",
            "Preparing to unpack .../28-cuda-cufft-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-cufft-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-cufft-dev-10-1.\n",
            "Preparing to unpack .../29-cuda-cufft-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-cufft-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-curand-10-1.\n",
            "Preparing to unpack .../30-cuda-curand-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-curand-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-curand-dev-10-1.\n",
            "Preparing to unpack .../31-cuda-curand-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-curand-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-cusparse-10-1.\n",
            "Preparing to unpack .../32-cuda-cusparse-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-cusparse-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-cusparse-dev-10-1.\n",
            "Preparing to unpack .../33-cuda-cusparse-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-cusparse-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-npp-10-1.\n",
            "Preparing to unpack .../34-cuda-npp-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-npp-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-npp-dev-10-1.\n",
            "Preparing to unpack .../35-cuda-npp-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-npp-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvml-dev-10-1.\n",
            "Preparing to unpack .../36-cuda-nvml-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvml-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvjpeg-10-1.\n",
            "Preparing to unpack .../37-cuda-nvjpeg-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvjpeg-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvjpeg-dev-10-1.\n",
            "Preparing to unpack .../38-cuda-nvjpeg-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvjpeg-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package nsight-compute-2019.4.0.\n",
            "Preparing to unpack .../39-nsight-compute-2019.4.0_2019.4.0.12-1_amd64.deb ...\n",
            "Unpacking nsight-compute-2019.4.0 (2019.4.0.12-1) ...\n",
            "Selecting previously unselected package cuda-nsight-compute-10-1.\n",
            "Preparing to unpack .../40-cuda-nsight-compute-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nsight-compute-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package nsight-systems-2019.3.7.\n",
            "Preparing to unpack .../41-nsight-systems-2019.3.7_2019.3.7.5-3837e03_amd64.deb ...\n",
            "Unpacking nsight-systems-2019.3.7 (2019.3.7.5-3837e03) ...\n",
            "Selecting previously unselected package cuda-nsight-systems-10-1.\n",
            "Preparing to unpack .../42-cuda-nsight-systems-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nsight-systems-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvgraph-10-1.\n",
            "Preparing to unpack .../43-cuda-nvgraph-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvgraph-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-nvgraph-dev-10-1.\n",
            "Preparing to unpack .../44-cuda-nvgraph-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-nvgraph-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-visual-tools-10-1.\n",
            "Preparing to unpack .../45-cuda-visual-tools-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-visual-tools-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-tools-10-1.\n",
            "Preparing to unpack .../46-cuda-tools-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-tools-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-samples-10-1.\n",
            "Preparing to unpack .../47-cuda-samples-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-samples-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-documentation-10-1.\n",
            "Preparing to unpack .../48-cuda-documentation-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-documentation-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-libraries-dev-10-1.\n",
            "Preparing to unpack .../49-cuda-libraries-dev-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-libraries-dev-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-toolkit-10-1.\n",
            "Preparing to unpack .../50-cuda-toolkit-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-toolkit-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-libraries-10-1.\n",
            "Preparing to unpack .../51-cuda-libraries-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-libraries-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-runtime-10-1.\n",
            "Preparing to unpack .../52-cuda-runtime-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-runtime-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-demo-suite-10-1.\n",
            "Preparing to unpack .../53-cuda-demo-suite-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-demo-suite-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda-10-1.\n",
            "Preparing to unpack .../54-cuda-10-1_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda-10-1 (10.1.243-1) ...\n",
            "Selecting previously unselected package cuda.\n",
            "Preparing to unpack .../55-cuda_10.1.243-1_amd64.deb ...\n",
            "Unpacking cuda (10.1.243-1) ...\n",
            "Selecting previously unselected package libasan3:amd64.\n",
            "Preparing to unpack .../56-libasan3_6.5.0-2ubuntu1~18.04_amd64.deb ...\n",
            "Unpacking libasan3:amd64 (6.5.0-2ubuntu1~18.04) ...\n",
            "Selecting previously unselected package libgcc-6-dev:amd64.\n",
            "Preparing to unpack .../57-libgcc-6-dev_6.5.0-2ubuntu1~18.04_amd64.deb ...\n",
            "Unpacking libgcc-6-dev:amd64 (6.5.0-2ubuntu1~18.04) ...\n",
            "Selecting previously unselected package gcc-6.\n",
            "Preparing to unpack .../58-gcc-6_6.5.0-2ubuntu1~18.04_amd64.deb ...\n",
            "Unpacking gcc-6 (6.5.0-2ubuntu1~18.04) ...\n",
            "Selecting previously unselected package libstdc++-6-dev:amd64.\n",
            "Preparing to unpack .../59-libstdc++-6-dev_6.5.0-2ubuntu1~18.04_amd64.deb ...\n",
            "Unpacking libstdc++-6-dev:amd64 (6.5.0-2ubuntu1~18.04) ...\n",
            "Selecting previously unselected package g++-6.\n",
            "Preparing to unpack .../60-g++-6_6.5.0-2ubuntu1~18.04_amd64.deb ...\n",
            "Unpacking g++-6 (6.5.0-2ubuntu1~18.04) ...\n",
            "Processing triggers for mime-support (3.60ubuntu1) ...\n",
            "Setting up nsight-compute-2019.4.0 (2019.4.0.12-1) ...\n",
            "Setting up gcc-6-base:amd64 (6.5.0-2ubuntu1~18.04) ...\n",
            "Setting up cuda-license-10-1 (10.1.243-1) ...\n",
            "*** LICENSE AGREEMENT ***\n",
            "By using this software you agree to fully comply with the terms and \n",
            "conditions of the EULA (End User License Agreement). The EULA is located\n",
            "at /usr/local/cuda-10.1/doc/EULA.txt. The EULA can also be found at\n",
            "http://docs.nvidia.com/cuda/eula/index.html. If you do not agree to the\n",
            "terms and conditions of the EULA, do not use the software.\n",
            "\n",
            "Setting up cuda-cusparse-10-1 (10.1.243-1) ...\n",
            "Setting up nsight-systems-2019.3.7 (2019.3.7.5-3837e03) ...\n",
            "update-alternatives: using /opt/nvidia/nsight-systems/2019.3.7/Target-x86_64/x86_64/nsys to provide /usr/local/bin/nsys (nsys) in auto mode\n",
            "update-alternatives: using /opt/nvidia/nsight-systems/2019.3.7/Host-x86_64/nsight-sys to provide /usr/local/bin/nsight-sys (nsight-sys) in auto mode\n",
            "update-alternatives: using /opt/nvidia/nsight-systems/2019.3.7/Host-x86_64/nsys-exporter to provide /usr/local/bin/nsys-exporter (nsys-exporter) in auto mode\n",
            "Setting up cuda-nvjpeg-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nsight-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nvdisasm-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nvml-dev-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-sanitizer-api-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nvjpeg-dev-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-cufft-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nvprof-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nvrtc-10-1 (10.1.243-1) ...\n",
            "Setting up libcublas10 (10.2.1.243-1) ...\n",
            "Setting up libcublas-dev (10.2.1.243-1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1) ...\n",
            "Setting up cuda-npp-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-cufft-dev-10-1 (10.1.243-1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Setting up cuda-nvvp-10-1 (10.1.243-1) ...\n",
            "Setting up cpp-6 (6.5.0-2ubuntu1~18.04) ...\n",
            "Setting up cuda-driver-dev-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-cuobjdump-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nsight-systems-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-gdb-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nsight-compute-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-cudart-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-npp-dev-10-1 (10.1.243-1) ...\n",
            "Setting up libasan3:amd64 (6.5.0-2ubuntu1~18.04) ...\n",
            "Setting up libgcc-6-dev:amd64 (6.5.0-2ubuntu1~18.04) ...\n",
            "Setting up libstdc++-6-dev:amd64 (6.5.0-2ubuntu1~18.04) ...\n",
            "Setting up cuda-cusolver-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-misc-headers-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nvtx-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-curand-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-cusparse-dev-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-gpu-library-advisor-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nvgraph-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nvprune-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-memcheck-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-cudart-dev-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nvrtc-dev-10-1 (10.1.243-1) ...\n",
            "Setting up gcc-6 (6.5.0-2ubuntu1~18.04) ...\n",
            "Setting up g++-6 (6.5.0-2ubuntu1~18.04) ...\n",
            "Setting up cuda-curand-dev-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-cusolver-dev-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nvgraph-dev-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-libraries-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-nvcc-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-runtime-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-compiler-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-demo-suite-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-visual-tools-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-libraries-dev-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-samples-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-cupti-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-documentation-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-command-line-tools-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-tools-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-toolkit-10-1 (10.1.243-1) ...\n",
            "Setting up cuda-10-1 (10.1.243-1) ...\n",
            "Setting up cuda (10.1.243-1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1) ...\n",
            "   Cloning default registries into `~/.julia`\n",
            "   Cloning registry from \"https://github.com/JuliaRegistries/General.git\"\n",
            "\u001b[2K\u001b[?25h     Added registry `General` to `~/.julia/registries/General`\n",
            " Resolving package versions...\n",
            " Installed VersionParsing ── v1.1.3\n",
            " Installed ZMQ ───────────── v1.0.0\n",
            " Installed Parsers ───────── v0.3.7\n",
            " Installed IJulia ────────── v1.20.0\n",
            " Installed Compat ────────── v2.2.0\n",
            " Installed BinaryProvider ── v0.5.7\n",
            " Installed JSON ──────────── v0.21.0\n",
            " Installed MbedTLS ───────── v0.7.0\n",
            " Installed SoftGlobalScope ─ v1.0.10\n",
            " Installed Conda ─────────── v1.3.0\n",
            "  Updating `~/.julia/environments/v1.2/Project.toml`\n",
            "  [7073ff75] + IJulia v1.20.0\n",
            "  Updating `~/.julia/environments/v1.2/Manifest.toml`\n",
            "  [b99e7846] + BinaryProvider v0.5.7\n",
            "  [34da2185] + Compat v2.2.0\n",
            "  [8f4d0f93] + Conda v1.3.0\n",
            "  [7073ff75] + IJulia v1.20.0\n",
            "  [682c06a0] + JSON v0.21.0\n",
            "  [739be429] + MbedTLS v0.7.0\n",
            "  [69de0a69] + Parsers v0.3.7\n",
            "  [b85f4697] + SoftGlobalScope v1.0.10\n",
            "  [81def892] + VersionParsing v1.1.3\n",
            "  [c2297ded] + ZMQ v1.0.0\n",
            "  [2a0f44e3] + Base64 \n",
            "  [ade2ca70] + Dates \n",
            "  [8bb1440f] + DelimitedFiles \n",
            "  [8ba89e20] + Distributed \n",
            "  [7b1f6079] + FileWatching \n",
            "  [b77e0a4c] + InteractiveUtils \n",
            "  [76f85450] + LibGit2 \n",
            "  [8f399da3] + Libdl \n",
            "  [37e2e46d] + LinearAlgebra \n",
            "  [56ddb016] + Logging \n",
            "  [d6f4376e] + Markdown \n",
            "  [a63ad114] + Mmap \n",
            "  [44cfe95a] + Pkg \n",
            "  [de0858da] + Printf \n",
            "  [3fa0cd96] + REPL \n",
            "  [9a3f8284] + Random \n",
            "  [ea8e919c] + SHA \n",
            "  [9e88b42a] + Serialization \n",
            "  [1a1011a3] + SharedArrays \n",
            "  [6462fe0b] + Sockets \n",
            "  [2f01184e] + SparseArrays \n",
            "  [10745b16] + Statistics \n",
            "  [8dfed614] + Test \n",
            "  [cf7118a7] + UUIDs \n",
            "  [4ec0a83e] + Unicode \n",
            "  Building Conda ──→ `~/.julia/packages/Conda/kLXeC/deps/build.log`\n",
            "  Building ZMQ ────→ `~/.julia/packages/ZMQ/ABGOx/deps/build.log`\n",
            "  Building MbedTLS → `~/.julia/packages/MbedTLS/a1JFn/deps/build.log`\n",
            "  Building IJulia ─→ `~/.julia/packages/IJulia/fRegO/deps/build.log`\n",
            " Resolving package versions...\n",
            " Installed URIParser ────────── v0.4.0\n",
            " Installed AbstractFFTs ─────── v0.4.1\n",
            " Installed Adapt ────────────── v1.0.0\n",
            " Installed Tokenize ─────────── v0.5.6\n",
            " Installed CUDAnative ───────── v2.4.0\n",
            " Installed JLD2 ─────────────── v0.1.3\n",
            " Installed CuArrays ─────────── v1.3.0\n",
            " Installed Requires ─────────── v0.5.2\n",
            " Installed CUDAdrv ──────────── v3.1.0\n",
            " Installed CSTParser ────────── v1.0.0\n",
            " Installed Reexport ─────────── v0.2.0\n",
            " Installed TimerOutputs ─────── v0.5.0\n",
            " Installed OrderedCollections ─ v1.1.0\n",
            " Installed Knet ─────────────── v1.3.0\n",
            " Installed CUDAapi ──────────── v1.2.0\n",
            " Installed NNlib ────────────── v0.6.0\n",
            " Installed GPUArrays ────────── v1.0.4\n",
            " Installed FillArrays ───────── v0.7.4\n",
            " Installed DataStructures ───── v0.17.5\n",
            " Installed Crayons ──────────── v4.0.0\n",
            " Installed TranscodingStreams ─ v0.9.5\n",
            " Installed CodecZlib ────────── v0.6.0\n",
            " Installed CEnum ────────────── v0.2.0\n",
            " Installed FFTW ─────────────── v1.0.1\n",
            " Installed BinDeps ──────────── v0.8.10\n",
            " Installed MacroTools ───────── v0.5.1\n",
            " Installed SpecialFunctions ─── v0.8.0\n",
            " Installed LLVM ─────────────── v1.3.2\n",
            " Installed AutoGrad ─────────── v1.2.0\n",
            " Installed FileIO ───────────── v1.0.7\n",
            "  Updating `~/.julia/environments/v1.2/Project.toml`\n",
            "  [1902f260] + Knet v1.3.0\n",
            "  Updating `~/.julia/environments/v1.2/Manifest.toml`\n",
            "  [621f4979] + AbstractFFTs v0.4.1\n",
            "  [79e6a3ab] + Adapt v1.0.0\n",
            "  [6710c13c] + AutoGrad v1.2.0\n",
            "  [9e28174c] + BinDeps v0.8.10\n",
            "  [fa961155] + CEnum v0.2.0\n",
            "  [00ebfdb7] + CSTParser v1.0.0\n",
            "  [3895d2a7] + CUDAapi v1.2.0\n",
            "  [c5f51814] + CUDAdrv v3.1.0\n",
            "  [be33ccc6] + CUDAnative v2.4.0\n",
            "  [944b1d66] + CodecZlib v0.6.0\n",
            "  [a8cc5b0e] + Crayons v4.0.0\n",
            "  [3a865a2d] + CuArrays v1.3.0\n",
            "  [864edb3b] + DataStructures v0.17.5\n",
            "  [7a1cc6ca] + FFTW v1.0.1\n",
            "  [5789e2e9] + FileIO v1.0.7\n",
            "  [1a297f60] + FillArrays v0.7.4\n",
            "  [0c68f7d7] + GPUArrays v1.0.4\n",
            "  [033835bb] + JLD2 v0.1.3\n",
            "  [1902f260] + Knet v1.3.0\n",
            "  [929cbde3] + LLVM v1.3.2\n",
            "  [1914dd2f] + MacroTools v0.5.1\n",
            "  [872c559c] + NNlib v0.6.0\n",
            "  [bac558e1] + OrderedCollections v1.1.0\n",
            "  [189a3867] + Reexport v0.2.0\n",
            "  [ae029012] + Requires v0.5.2\n",
            "  [276daf66] + SpecialFunctions v0.8.0\n",
            "  [a759f4b9] + TimerOutputs v0.5.0\n",
            "  [0796e94c] + Tokenize v0.5.6\n",
            "  [3bb67fe8] + TranscodingStreams v0.9.5\n",
            "  [30578b45] + URIParser v0.4.0\n",
            "  Building SpecialFunctions → `~/.julia/packages/SpecialFunctions/ne2iw/deps/build.log`\n",
            "  Building FFTW ────────────→ `~/.julia/packages/FFTW/MJ7kl/deps/build.log`\n",
            "  Building CodecZlib ───────→ `~/.julia/packages/CodecZlib/5t9zO/deps/build.log`\n",
            "  Building Knet ────────────→ `~/.julia/packages/Knet/ALUfa/deps/build.log`\n",
            "Precompiling project...\n",
            "Precompiling Knet\n",
            "Precompiling IJulia\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QvCZH_l6Kb0N",
        "colab_type": "text"
      },
      "source": [
        "### After executing the first cell click on `connect to hosted runtime` which you can find in the drop menu on the upper right corner on the left of Editing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5jhycwF5_ZST",
        "colab_type": "code",
        "outputId": "53c0b87e-80a4-4e93-ecf9-0eb9007dcee0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 399
        }
      },
      "source": [
        "import Pkg\n",
        "Pkg.add(\"IterTools\")\n",
        "Pkg.add(\"AutoGrad\")\n",
        "using Pkg\n",
        "using Knet, Test, Base.Iterators, IterTools, Random # , LinearAlgebra, StatsBase\n",
        "using AutoGrad: @gcheck  # to check gradients, use with Float64\n",
        "\n",
        "macro size(z, s) # for debugging\n",
        "    esc(:(@assert (size($z) == $s) string(summary($z), !=, $s))) # for debugging\n",
        "end # for debugging\n",
        "\n",
        "Pkg.add(\"CuArrays\")\n",
        "Pkg.build(\"CuArrays\")\n",
        "\n",
        "using CuArrays: CuArrays, usage_limit"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m registry at `~/.julia/registries/General`\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m git-repo `https://github.com/JuliaRegistries/General.git`\n",
            "\u001b[?25l\u001b[2K\u001b[?25h\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n",
            "\u001b[32m\u001b[1m Installed\u001b[22m\u001b[39m IterTools ─ v1.2.0\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Project.toml`\n",
            " \u001b[90m [c8e1da08]\u001b[39m\u001b[92m + IterTools v1.2.0\u001b[39m\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Manifest.toml`\n",
            " \u001b[90m [c8e1da08]\u001b[39m\u001b[92m + IterTools v1.2.0\u001b[39m\n",
            "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Project.toml`\n",
            " \u001b[90m [6710c13c]\u001b[39m\u001b[92m + AutoGrad v1.2.0\u001b[39m\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Manifest.toml`\n",
            "\u001b[90m [no changes]\u001b[39m\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "┌ Info: Precompiling IterTools [c8e1da08-722c-5040-9ed9-7db0dc04731e]\n",
            "└ @ Base loading.jl:1242\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Project.toml`\n",
            " \u001b[90m [3a865a2d]\u001b[39m\u001b[92m + CuArrays v1.3.0\u001b[39m\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Manifest.toml`\n",
            "\u001b[90m [no changes]\u001b[39m\n",
            "\u001b[32m\u001b[1m  Building\u001b[22m\u001b[39m Conda → `~/.julia/packages/Conda/kLXeC/deps/build.log`\n",
            "\u001b[32m\u001b[1m  Building\u001b[22m\u001b[39m FFTW ─→ `~/.julia/packages/FFTW/MJ7kl/deps/build.log`\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BHrO1rqI1KLr",
        "colab_type": "code",
        "outputId": "f22f241a-b6cd-45af-a1d1-f636771e45a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        }
      },
      "source": [
        "Pkg.add(\"Statistics\")\n",
        "import Statistics\n",
        "using Statistics"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Project.toml`\n",
            " \u001b[90m [10745b16]\u001b[39m\u001b[92m + Statistics \u001b[39m\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Manifest.toml`\n",
            "\u001b[90m [no changes]\u001b[39m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C4cuTVpwmo1x",
        "colab_type": "text"
      },
      "source": [
        "### After setting up the environment, check what we have"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SEBWflYFlx2v",
        "colab_type": "code",
        "outputId": "653cb76f-4e17-4dfb-def7-a3278f2be68b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 399
        }
      },
      "source": [
        "Pkg.update()\n",
        "pkgs = Pkg.installed()\n",
        "\n",
        "for package in keys(pkgs)\n",
        "    if pkgs[package] == nothing\n",
        "        pkgs[package] = VersionNumber(\"0.0.1\")\n",
        "    end\n",
        "    println(\"Package name: \", package, \" Version: \", pkgs[package])\n",
        "end"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m registry at `~/.julia/registries/General`\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m git-repo `https://github.com/JuliaRegistries/General.git`\n",
            "\u001b[?25l\u001b[2K\u001b[?25h\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n",
            "\u001b[32m\u001b[1m Installed\u001b[22m\u001b[39m CUDAdrv ──── v4.0.1\n",
            "\u001b[32m\u001b[1m Installed\u001b[22m\u001b[39m CUDAnative ─ v2.5.0\n",
            "\u001b[32m\u001b[1m Installed\u001b[22m\u001b[39m CuArrays ─── v1.4.0\n",
            "\u001b[32m\u001b[1m Installed\u001b[22m\u001b[39m Knet ─────── v1.2.7\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Project.toml`\n",
            " \u001b[90m [3a865a2d]\u001b[39m\u001b[93m ↑ CuArrays v1.3.0 ⇒ v1.4.0\u001b[39m\n",
            " \u001b[90m [1902f260]\u001b[39m\u001b[95m ↓ Knet v1.3.0 ⇒ v1.2.7\u001b[39m\n",
            "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Manifest.toml`\n",
            " \u001b[90m [c5f51814]\u001b[39m\u001b[93m ↑ CUDAdrv v3.1.0 ⇒ v4.0.1\u001b[39m\n",
            " \u001b[90m [be33ccc6]\u001b[39m\u001b[93m ↑ CUDAnative v2.4.0 ⇒ v2.5.0\u001b[39m\n",
            " \u001b[90m [3a865a2d]\u001b[39m\u001b[93m ↑ CuArrays v1.3.0 ⇒ v1.4.0\u001b[39m\n",
            " \u001b[90m [1902f260]\u001b[39m\u001b[95m ↓ Knet v1.3.0 ⇒ v1.2.7\u001b[39m\n",
            "\u001b[32m\u001b[1m  Building\u001b[22m\u001b[39m Knet → `~/.julia/packages/Knet/IIjk8/deps/build.log`\n",
            "Package name: AutoGrad Version: 1.2.0\n",
            "Package name: IterTools Version: 1.2.0\n",
            "Package name: Statistics Version: 0.0.1\n",
            "Package name: Knet Version: 1.2.7\n",
            "Package name: CuArrays Version: 1.4.0\n",
            "Package name: IJulia Version: 1.20.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OxR_riRTnBA6",
        "colab_type": "text"
      },
      "source": [
        "### Set some constants"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zu7EphRom9F5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "fb28784b-a7ee-4138-ef3e-159103c476a9"
      },
      "source": [
        "#CuArrays.usage_limit[] = 9_000_000_000\n",
        "#array_type = KnetArray # For GPU instances\n",
        "Knet.atype() = KnetArray{Float32}  # determines what Knet.param() uses.\n",
        "BATCH_SIZE = 64"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j8_h7qRDnKrV",
        "colab_type": "text"
      },
      "source": [
        "### Part -1: Copy what we had\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nEukhjawnHRu",
        "colab_type": "code",
        "outputId": "a422cd9e-50c8-494a-c8dc-8a7390826ade",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "struct Vocab\n",
        "    w2i::Dict{String,Int}\n",
        "    i2w::Vector{String}\n",
        "    unk::Int\n",
        "    eos::Int\n",
        "    tokenizer\n",
        "end\n",
        "\n",
        "function Vocab(\n",
        "    file::String;\n",
        "    tokenizer = split,\n",
        "    vocabsize = Inf,\n",
        "    mincount = 1,\n",
        "    unk = \"<unk>\",\n",
        "    eos = \"<s>\",\n",
        ")\n",
        "    vocab_freq = Dict{String,Int64}(unk => 1, eos => 1)\n",
        "    w2i = Dict{String,Int64}(unk => 2, eos => 1)\n",
        "    i2w = Vector{String}()\n",
        "\n",
        "    push!(i2w, eos)\n",
        "    push!(i2w, unk)\n",
        "\n",
        "    open(file) do f\n",
        "        for line in eachline(f)\n",
        "            sentence = strip(lowercase(line))\n",
        "            sentence = tokenizer(line, [' '], keepempty = false)\n",
        "\n",
        "            for word in sentence\n",
        "                word == unk && continue\n",
        "                word == eos && continue # They are default ones to be added later\n",
        "                vocab_freq[word] = get!(vocab_freq, word, 0) + 1\n",
        "            end\n",
        "        end\n",
        "        close(f)\n",
        "    end\n",
        "\n",
        "\n",
        "    # End of vanilla implementation of the vocaulary\n",
        "    # From here we must add the mincount and vocabsize properties\n",
        "    # We must change the first two property of the vocab wrt those paramaters\n",
        "    vocab_freq = sort!(\n",
        "        collect(vocab_freq),\n",
        "        by = tuple -> last(tuple),\n",
        "        rev = true,\n",
        "    )\n",
        "\n",
        "    if length(vocab_freq) > vocabsize - 2 # eos and unk ones\n",
        "        vocab_freq = vocab_freq[1:vocabsize-2] # trim to fit the size\n",
        "    end\n",
        "\n",
        "    #vocab_freq = reverse(vocab_freq)\n",
        "\n",
        "    while true\n",
        "        length(vocab_freq) == 0 && break\n",
        "        word, freq = vocab_freq[end]\n",
        "        freq >= mincount && break # since it is already ordered\n",
        "        vocab_freq = vocab_freq[1:(end-1)]\n",
        "    end\n",
        "    #pushfirst!(vocab_freq,unk=>1,eos=>1) # freq does not matter, just adding the\n",
        "    for i = 1:length(vocab_freq)\n",
        "        word, freq = vocab_freq[i]\n",
        "        ind = (get!(w2i, word, 1 + length(w2i)))\n",
        "        (length(i2w) < ind) && push!(i2w, word)\n",
        "    end\n",
        "\n",
        "    return Vocab(w2i, i2w, 2, 1, tokenizer)\n",
        "end\n",
        "\n",
        "struct TextReader\n",
        "    file::String\n",
        "    vocab::Vocab\n",
        "end\n",
        "\n",
        "word2ind(dict, x) = get(dict, x, 2)  # Unk as 2\n",
        "\n",
        "#Implementing the iterate function\n",
        "function Base.iterate(r::TextReader, s = nothing)\n",
        "    if s == nothing\n",
        "        state = open(r.file)\n",
        "        Base.iterate(r, state)\n",
        "    else\n",
        "        if eof(s) == true\n",
        "            close(s)\n",
        "            return nothing\n",
        "        else\n",
        "            line = readline(s)\n",
        "            line = strip(lowercase(line))\n",
        "            sent = r.vocab.tokenizer(line, [' '], keepempty = false)\n",
        "            sent_ind = Int[]\n",
        "            for word in sent\n",
        "                ind = word2ind(r.vocab.w2i, word)\n",
        "                push!(sent_ind, ind)\n",
        "            end\n",
        "            return (sent_ind, s)\n",
        "        end\n",
        "    end\n",
        "end\n",
        "\n",
        "\n",
        "Base.IteratorSize(::Type{TextReader}) = Base.SizeUnknown()\n",
        "Base.IteratorEltype(::Type{TextReader}) = Base.HasEltype()\n",
        "Base.eltype(::Type{TextReader}) = Vector{Int}\n",
        "\n",
        "#Embed\n",
        "struct Embed\n",
        "    w\n",
        "end\n",
        "\n",
        "function Embed(vocabsize::Int, embedsize::Int)\n",
        "    Embed(param(embedsize, vocabsize))\n",
        "end\n",
        "\n",
        "function (l::Embed)(x)\n",
        "    l.w[:, x]\n",
        "end\n",
        "\n",
        "#Linear\n",
        "struct Linear\n",
        "    w\n",
        "    b\n",
        "end\n",
        "\n",
        "function Linear(inputsize::Int, outputsize::Int)\n",
        "    Linear(\n",
        "        param(outputsize, inputsize),\n",
        "        param0(outputsize),\n",
        "    )\n",
        "end\n",
        "\n",
        "function (l::Linear)(x)\n",
        "    l.w * mat(x, dims = 1) .+ l.b\n",
        "end\n",
        "\n",
        "# Mask!\n",
        "function mask!(a, pad)\n",
        "    matr = a\n",
        "    for j = 1:size(matr)[1]\n",
        "        i = 0\n",
        "        while (i < length(matr[j, :]) - 1)\n",
        "            if matr[j, length(matr[j, :])-i-1] != pad\n",
        "                break\n",
        "\n",
        "            elseif matr[j, length(matr[j, :])-i] == pad\n",
        "                matr[j, length(matr[j, :])-i] = 0\n",
        "            end\n",
        "            i += 1\n",
        "        end\n",
        "    end\n",
        "    return matr\n",
        "end"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "mask! (generic function with 1 method)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3eG_1TwVnWT4",
        "colab_type": "text"
      },
      "source": [
        "### Part 0: Load Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J20XbgRknUEH",
        "colab_type": "code",
        "outputId": "50cee37a-e3b3-4330-e438-aa009667eb1a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "datadir = \"datasets/tr_to_en\"\n",
        "\n",
        "if !isdir(datadir)\n",
        "    download(\n",
        "        \"http://www.phontron.com/data/qi18naacl-dataset.tar.gz\",\n",
        "        \"qi18naacl-dataset.tar.gz\",\n",
        "    )\n",
        "    run(`tar xzf qi18naacl-dataset.tar.gz`)\n",
        "end\n",
        "\n",
        "if !isdefined(Main, :tr_vocab)\n",
        "    tr_vocab = Vocab(\"$datadir/tr.train\", mincount = 5)\n",
        "    en_vocab = Vocab(\"$datadir/en.train\", mincount = 5)\n",
        "    tr_train = TextReader(\"$datadir/tr.train\", tr_vocab)\n",
        "    en_train = TextReader(\"$datadir/en.train\", en_vocab)\n",
        "    tr_dev = TextReader(\"$datadir/tr.dev\", tr_vocab)\n",
        "    en_dev = TextReader(\"$datadir/en.dev\", en_vocab)\n",
        "    tr_test = TextReader(\"$datadir/tr.test\", tr_vocab)\n",
        "    en_test = TextReader(\"$datadir/en.test\", en_vocab)\n",
        "    @info \"Testing data\"\n",
        "    @test length(tr_vocab.i2w) == 38126\n",
        "    @test length(first(tr_test)) == 16\n",
        "    @test length(collect(tr_test)) == 5029\n",
        "end"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "┌ Info: Testing data\n",
            "└ @ Main In[7]:20\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\u001b[32m\u001b[1mTest Passed\u001b[22m\u001b[39m"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HfiPwnAynfLQ",
        "colab_type": "text"
      },
      "source": [
        "### Part 1 - Minibatching"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "12DfXW0FneNu",
        "colab_type": "code",
        "outputId": "8eb29467-9583-434c-bf09-817104af1859",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "struct MTData\n",
        "    src::TextReader        # reader for source language data\n",
        "    tgt::TextReader        # reader for target language data\n",
        "    batchsize::Int         # desired batch size\n",
        "    maxlength::Int         # skip if source sentence above maxlength\n",
        "    batchmajor::Bool       # batch dims (B,T) if batchmajor=false (default) or (T,B) if true.\n",
        "    bucketwidth::Int       # batch sentences with length within bucketwidth of each other\n",
        "    buckets::Vector        # sentences collected in separate arrays called buckets for each length range\n",
        "    batchmaker::Function   # function that turns a bucket into a batch.\n",
        "end\n",
        "\n",
        "function MTData(\n",
        "    src::TextReader,\n",
        "    tgt::TextReader;\n",
        "    batchmaker = arraybatch,\n",
        "    batchsize = BATCH_SIZE,\n",
        "    maxlength = typemax(Int),\n",
        "    batchmajor = false,\n",
        "    bucketwidth = 10,\n",
        "    numbuckets = min(BATCH_SIZE, maxlength ÷ bucketwidth),\n",
        ")\n",
        "    buckets = [[] for i = 1:numbuckets] # buckets[i] is an array of sentence pairs with similar length\n",
        "    MTData(\n",
        "        src,\n",
        "        tgt,\n",
        "        batchsize,\n",
        "        maxlength,\n",
        "        batchmajor,\n",
        "        bucketwidth,\n",
        "        buckets,\n",
        "        batchmaker,\n",
        "    )\n",
        "end\n",
        "\n",
        "Base.IteratorSize(::Type{MTData}) = Base.SizeUnknown()\n",
        "Base.IteratorEltype(::Type{MTData}) = Base.HasEltype()\n",
        "Base.eltype(::Type{MTData}) = NTuple{2}\n",
        "\n",
        "function Base.iterate(d::MTData, state = nothing)\n",
        "    if state == nothing\n",
        "        for b in d.buckets\n",
        "            empty!(b)\n",
        "        end\n",
        "        state_src, state_tgt = nothing, nothing\n",
        "    else\n",
        "        state_src, state_tgt = state\n",
        "    end\n",
        "    bucket, ibucket = nothing, nothing\n",
        "\n",
        "\n",
        "    while true\n",
        "        iter_src = (state_src === nothing ? iterate(d.src) :\n",
        "                    iterate(d.src, state_src))\n",
        "        iter_tgt = (state_tgt === nothing ? iterate(d.tgt) :\n",
        "                    iterate(d.tgt, state_tgt))\n",
        "\n",
        "        if iter_src === nothing\n",
        "            ibucket = findfirst(x -> !isempty(x), d.buckets)\n",
        "            bucket = (ibucket === nothing ? nothing : d.buckets[ibucket])\n",
        "            break\n",
        "        else\n",
        "            sent_src, state_src = iter_src\n",
        "            sent_tgt, state_tgt = iter_tgt\n",
        "            if length(sent_src) > d.maxlength || length(sent_src) == 0\n",
        "                continue\n",
        "            end\n",
        "            ibucket = min(\n",
        "                1 + (length(sent_src) - 1) ÷ d.bucketwidth,\n",
        "                length(d.buckets),\n",
        "            )\n",
        "            bucket = d.buckets[ibucket]\n",
        "            push!(bucket, (sent_src, sent_tgt))\n",
        "            if length(bucket) === d.batchsize\n",
        "                break\n",
        "            end\n",
        "        end\n",
        "    end\n",
        "    if bucket === nothing\n",
        "        return nothing\n",
        "    end\n",
        "\n",
        "    batch = d.batchmaker(d, bucket)\n",
        "\n",
        "    empty!(bucket)\n",
        "    return batch, (state_src, state_tgt)\n",
        "end\n",
        "\n",
        "\n",
        "function arraybatch(d::MTData, bucket)\n",
        "    bucketx = map(x -> x[1], bucket)\n",
        "    buckety = map(x -> x[2], bucket)\n",
        "    batch_x = fill(d.src.vocab.eos, length(bucketx), maximum(length.(bucketx)))\n",
        "    for i = 1:length(bucket)\n",
        "        batch_x[i, end-length(bucketx[i])+1:end] = bucketx[i]\n",
        "    end\n",
        "    batch_y = fill(\n",
        "        d.tgt.vocab.eos,\n",
        "        length(buckety),\n",
        "        maximum(length.(buckety)) + 2,\n",
        "    )\n",
        "    for i = 1:length(bucket)\n",
        "        batch_y[i, 2:length(buckety[i])+1] = buckety[i]\n",
        "    end\n",
        "\n",
        "    return (batch_x, batch_y)\n",
        "end\n",
        "\n",
        "#-\n",
        "\n",
        "@info \"Testing MTData\"\n",
        "dtrn = MTData(tr_train, en_train)\n",
        "ddev = MTData(tr_dev, en_dev)\n",
        "dtst = MTData(tr_test, en_test)\n",
        "\n",
        "x, y = first(dtst)\n",
        "\n",
        "#@test length(collect(dtst)) == 48\n",
        "#@test size.((x, y)) == ((128, 10), (128, 24))\n",
        "@test x[1, 1] == tr_vocab.eos\n",
        "@test x[1, end] != tr_vocab.eos\n",
        "@test y[1, 1] == en_vocab.eos\n",
        "@test y[1, 2] != en_vocab.eos\n",
        "@test y[1, end] == en_vocab.eos"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "┌ Info: Testing MTData\n",
            "└ @ Main In[8]:110\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\u001b[32m\u001b[1mTest Passed\u001b[22m\u001b[39m"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1RL83XVtnnvE",
        "colab_type": "text"
      },
      "source": [
        "## Part 2 - S2S w/o attention"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ner4DkeXnrrE",
        "colab_type": "code",
        "outputId": "99c9f7b2-2ba1-4e3a-8cbf-100db938bf0c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "struct S2S_v1\n",
        "    srcembed::Embed     # source language embedding\n",
        "    encoder::RNN        # encoder RNN (can be bidirectional)\n",
        "    tgtembed::Embed     # target language embedding\n",
        "    decoder::RNN        # decoder RNN\n",
        "    projection::Linear  # converts decoder output to vocab scores\n",
        "    dropout::Real       # dropout probability to prevent overfitting\n",
        "    srcvocab::Vocab     # source language vocabulary\n",
        "    tgtvocab::Vocab     # target language vocabulary\n",
        "end\n",
        "\n",
        "function S2S_v1(\n",
        "    hidden::Int,         # hidden size for both the encoder and decoder RNN\n",
        "    srcembsz::Int,       # embedding size for source language\n",
        "    tgtembsz::Int,       # embedding size for target language\n",
        "    srcvocab::Vocab,     # vocabulary for source language\n",
        "    tgtvocab::Vocab;     # vocabulary for target language\n",
        "    layers = 1,            # number of layers\n",
        "    bidirectional = false, # whether encoder RNN is bidirectional\n",
        "    dropout = 0,\n",
        ")           # dropout probability\n",
        "\n",
        "\n",
        "    layerMultiplier = bidirectional ? 2 : 1\n",
        "\n",
        "    S2S_v1(\n",
        "        Embed(length(srcvocab.i2w), srcembsz),\n",
        "        RNN(\n",
        "            srcembsz,\n",
        "            hidden,\n",
        "            numLayers = layers,\n",
        "            bidirectional = bidirectional,\n",
        "            dropout = dropout\n",
        "        ),\n",
        "        Embed(length(tgtvocab.i2w), tgtembsz),\n",
        "        RNN(\n",
        "            tgtembsz,\n",
        "            hidden,\n",
        "            numLayers = layerMultiplier * layers,\n",
        "            dropout = dropout\n",
        "        ),\n",
        "        Linear(hidden, length(tgtvocab.i2w)),\n",
        "        dropout,\n",
        "        srcvocab,\n",
        "        tgtvocab,\n",
        "    )\n",
        "\n",
        "end\n",
        "\n",
        "function (s::S2S_v1)(src, tgt; average = true)\n",
        "    #B,Tx = size(src,2)\n",
        "    B, Ty = size(tgt)\n",
        "    Ty -= 1 # Crop one\n",
        "    # Ex, Ey = length(model.srcembed([1])), length(model.tgtembed([1]))\n",
        "\n",
        "    rnn_encoder = s.encoder\n",
        "    rnn_decoder = s.decoder\n",
        "    project = s.projection\n",
        "\n",
        "    # Lx, Ly = rnn_encoder.numLayers, rnn_decoder.numLayers\n",
        "    # Hx, Hy = rnn_encoder.hiddenSize, rnn_decoder.hiddenSize\n",
        "    # Dx = Ly/Lx\n",
        "\n",
        "    emb_out_src = s.srcembed(src)\n",
        "    #@test size(emb_out_src)== (Ex,B,Tx) # Done\n",
        "\n",
        "    # Safe for repetitive usage\n",
        "    rnn_encoder.h = 0\n",
        "    rnn_encoder.c = 0\n",
        "\n",
        "    y_enc = rnn_encoder(dropout(emb_out_src,s.dropout))\n",
        "    #@test size(y_enc) == (Hx*Dx,B,Tx)\n",
        "    h_enc = rnn_encoder.h\n",
        "    #@test size(h_enc) == (Hx,B,Lx*Dx)\n",
        "    c_enc = rnn_encoder.c\n",
        "\n",
        "    emb_out_tgt = s.tgtembed(tgt[:, 1:end-1])\n",
        "    #@test size(emb_out_tgt)== (Ey,B,Ty)\n",
        "\n",
        "    rnn_decoder.h = h_enc\n",
        "    rnn_decoder.c = c_enc\n",
        "    y_dec = rnn_decoder(emb_out_tgt)\n",
        "    #@test size(y_dec)==(Hy,B,Ty)\n",
        "\n",
        "    project_inp = reshape(y_dec, :, B * Ty)\n",
        "    project_out = project(project_inp)\n",
        "\n",
        "    #@test size(project_out)==(length(project.b),B*Ty)\n",
        "\n",
        "    verify = deepcopy(tgt)\n",
        "    mask!(verify, s.tgtvocab.eos)\n",
        "\n",
        "    average && return mean(nll(project_out, verify[:, 2:end]))\n",
        "    return nll(project_out, verify[:, 2:end]; average = false)\n",
        "end\n",
        "\n",
        "#-\n",
        "\n",
        "@info \"Testing S2S_v1\"\n",
        "Knet.seed!(1)\n",
        "model = S2S_v1(\n",
        "    512,\n",
        "    512,\n",
        "    512,\n",
        "    tr_vocab,\n",
        "    en_vocab;\n",
        "    layers = 2,\n",
        "    bidirectional = true,\n",
        "    dropout = 0.2,\n",
        ")\n",
        "GC.gc();(x, y) = first(dtst)\n",
        "## Your loss can be slightly different due to different ordering of words in the vocabulary.\n",
        "## The reference vocabulary starts with eos, unk, followed by words in decreasing frequency.\n",
        "#@test model(x,y; average=false) == (14097.471f0, 1432)  !!!!!!\n",
        "GC.gc();println(model(x, y; average = false))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "┌ Info: Testing S2S_v1\n",
            "└ @ Main In[9]:99\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "(14471.718f0, 1470)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7UgA9ewdqOis",
        "colab_type": "text"
      },
      "source": [
        "### Loss Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bRp3ZKwGn1ih",
        "colab_type": "code",
        "outputId": "58a4612e-b728-47d7-f1b8-197a168f60f4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        }
      },
      "source": [
        "function loss(model, data; average = true)\n",
        "    total_loss = 0\n",
        "    total_word = 0\n",
        "\n",
        "    for (x, y) in collect(data)\n",
        "        curr_loss, curr_word = model(x, y; average = false)\n",
        "        total_loss += curr_loss\n",
        "        total_word += curr_word\n",
        "    end\n",
        "    if total_word == 0\n",
        "      average && return 0\n",
        "      return 0,0\n",
        "    end\n",
        "    average && return total_loss / total_word\n",
        "    return (total_loss, total_word)\n",
        "\n",
        "end\n",
        "\n",
        "#-\n",
        "\n",
        "@info \"Testing loss\"\n",
        "@time res = loss(model, dtst, average = false)\n",
        "println(res)\n",
        "#@test res == (1.0429117f6, 105937) !!!!!!!!!!"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "┌ Info: Testing loss\n",
            "└ @ Main In[10]:21\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  2.540305 seconds (1.97 M allocations: 193.435 MiB, 11.60% gc time)\n",
            "(1.04291994f6, 105937)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n_rOpgb8qllo",
        "colab_type": "text"
      },
      "source": [
        "### Train Function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KRvipEH3of_3",
        "colab_type": "code",
        "outputId": "17dd23a2-4bcb-4f2b-ae03-debb8de02bc1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "function train!(model, trn, dev, tst...)\n",
        "    bestmodel, bestloss = deepcopy(model), loss(model, dev;average=true)\n",
        "    progress!(adam(model, trn), steps = 100) do y\n",
        "        losses = [loss(model, d) for d in (dev, tst...)]\n",
        "        if losses[1] < bestloss\n",
        "            bestmodel, bestloss = deepcopy(model), losses[1]\n",
        "        end\n",
        "        return (losses...,)\n",
        "    end\n",
        "    return bestmodel\n",
        "end"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "train! (generic function with 1 method)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yj9vvO-7q_F1",
        "colab_type": "text"
      },
      "source": [
        "### Train Parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LhwkuDPtokVz",
        "colab_type": "code",
        "outputId": "a2f7fa02-1d5c-4fd3-e0b9-eb47c979d16e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 521
        }
      },
      "source": [
        "@info \"Training S2S_v1\"\n",
        "epochs = 10\n",
        "ctrn = collect(dtrn)\n",
        "trnx10 = collect(flatten(shuffle!(ctrn) for i = 1:epochs))\n",
        "trn20 = ctrn[1:20]\n",
        "dev38 = collect(ddev)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "┌ Info: Training S2S_v1\n",
            "└ @ Main In[12]:1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "69-element Array{Tuple{T,T} where T,1}:\n",
              " ([1 1 … 2 3; 1 1 … 2 3; … ; 1 1 … 553 3; 1 9 … 8 8], [1 43 … 1 1; 1 37 … 1 1; … ; 1 43 … 1 1; 1 23 … 1 1])                \n",
              " ([1000 1673 … 1066 3; 1 1 … 25 3; … ; 1 1 … 130 3; 1 7 … 2 3], [1 60 … 4 1; 1 9 … 1 1; … ; 1 11 … 1 1; 1 19 … 1 1])       \n",
              " ([1 1 … 2865 3; 1 1 … 2 3; … ; 1 1 … 2 3; 1 1 … 1618 3], [1 14 … 1 1; 1 158 … 1 1; … ; 1 29 … 1 1; 1 11 … 4 1])           \n",
              " ([1 1 … 827 3; 1 1 … 2 3; … ; 1 296 … 442 3; 1 1 … 2529 24], [1 20 … 1 1; 1 9 … 1 1; … ; 1 775 … 1 1; 1 41 … 1 1])        \n",
              " ([6 35117 … 1494 3; 43 15 … 10233 3; … ; 1 1 … 2528 3; 1 1 … 21 3], [1 6 … 1 1; 1 20 … 1 1; … ; 1 21 … 1 1; 1 21 … 1 1])  \n",
              " ([1 1 … 8575 3; 1 1 … 2 3; … ; 1 1 … 2785 3; 1 7 … 2329 3], [1 21 … 1 1; 1 13 … 1 1; … ; 1 6 … 1 1; 1 6 … 1 1])           \n",
              " ([1 54 … 630 3; 1 1 … 84 3; … ; 9 9 … 3 8; 1 1 … 587 3], [1 11 … 1 1; 1 37 … 1 1; … ; 1 23 … 1 1; 1 27 … 1 1])            \n",
              " ([1 1 … 2 3; 1 1 … 95 14; … ; 1 1 … 2 3; 1 1 … 337 3], [1 6 … 1 1; 1 75 … 1 1; … ; 1 6 … 1 1; 1 84 … 1 1])                \n",
              " ([1 1 … 25968 3; 1 1 … 337 3; … ; 1 1 … 2908 3; 1 104 … 6173 3], [1 11 … 1 1; 1 11 … 1 1; … ; 1 5 … 1 1; 1 14 … 1 1])     \n",
              " ([1 1 … 3 8; 1 1 … 2 3; … ; 1 1 … 2 3; 1 1 … 567 3], [1 23 … 1 1; 1 141 … 1 1; … ; 1 14 … 1 1; 1 13 … 1 1])               \n",
              " ([1 1 … 2 3; 1 1 … 16417 3; … ; 1 1 … 379 3; 1 1 … 23846 3], [1 64 … 1 1; 1 21 … 1 1; … ; 1 20 … 1 1; 1 6 … 1 1])         \n",
              " ([1 1 … 9343 3; 1 1 … 2216 14; … ; 4576 1233 … 2 3; 1 1 … 35 3], [1 20 … 1 1; 1 12 … 1 1; … ; 1 21 … 1 1; 1 12 … 1 1])    \n",
              " ([1 1 … 2 2458; 1 6 … 1804 3; … ; 1 17590 … 4211 3; 1 1 … 12296 3], [1 11 … 1 1; 1 6 … 1 1; … ; 1 7277 … 1 1; 1 20 … 1 1])\n",
              " ⋮                                                                                                                         \n",
              " ([1 1 … 14196 3; 1 1 … 3555 3; … ; 1 1 … 2 3; 1 1 … 130 3], [1 13 … 1 1; 1 170 … 1 1; … ; 1 61 … 1 1; 1 20 … 1 1])        \n",
              " ([1 1 … 24099 3; 6 2293 … 30855 3; … ; 1 1 … 198 3; 1 1 … 148 17], [1 295 … 1 1; 1 6 … 1 1; … ; 1 163 … 1 1; 1 57 … 1 1]) \n",
              " ([1 1 … 205 3; 1 7 … 5 3; … ; 1 1 … 171 3; 1 1 … 148 17], [1 2060 … 1 1; 1 5 … 1 1; … ; 1 163 … 1 1; 1 57 … 1 1])         \n",
              " ([1 1 … 3 8; 1 1 … 14422 3; … ; 1 1 … 19708 3; 1 1 … 2185 3], [1 23 … 1 1; 1 53 … 1 1; … ; 1 24 … 1 1; 1 5 … 1 1])        \n",
              " ([1 1 … 2 3; 1 6 … 2152 3; … ; 1 1 … 4565 3; 1 1 … 151 3], [1 27 … 1 1; 1 6 … 4 1; … ; 1 1637 … 1 1; 1 6 … 1 1])          \n",
              " ([1 1 … 3213 3; 1 1 … 2 3; … ; 1 1 … 1174 3; 9 848 … 3 8], [1 6 … 1 1; 1 2 … 1 1; … ; 1 2 … 1 1; 1 23 … 1 1])             \n",
              " ([1 1 … 2504 3; 1 1 … 8075 3; … ; 1 1 … 2 3; 1 1 … 182 3], [1 840 … 1 1; 1 6 … 4 1; … ; 1 12 … 1 1; 1 112 … 1 1])         \n",
              " ([9 9 … 3 8; 1 1 … 3 8; … ; 1 1 … 2549 3; 1 1 … 3 8], [1 23 … 1 1; 1 23 … 1 1; … ; 1 6 … 1 1; 1 23 … 1 1])                \n",
              " ([1 1 … 29092 3; 1 1 … 2099 24; … ; 1 1 … 3220 3; 1 1 … 86 3], [1 41 … 1 1; 1 11 … 1 1; … ; 1 27 … 1 1; 1 15 … 1 1])      \n",
              " ([1 1 … 3 8; 6 549 … 2 3580; … ; 1 1 … 17 8; 9 25 … 3 8], [1 23 … 1 1; 1 2655 … 1 1; … ; 1 23 … 1 1; 1 23 … 17 1])        \n",
              " ([1 1 … 3 8; 1 9 … 3 8; 5931 1070 … 756 3], [1 23 … 17 1; 1 23 … 1 1; 1 11 … 1 1])                                        \n",
              " ([9 275 … 2 8], [1 23 … 17 1])                                                                                            "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g4GR84F7rLZ6",
        "colab_type": "text"
      },
      "source": [
        "### Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h2zxIT2-pUFD",
        "colab_type": "code",
        "outputId": "ff8a1022-a577-4755-bb7d-10c9c174686c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "GC.gc();\n",
        "model = train!(model, trnx10, dev38, trn20)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "┣████████████████████┫ [100.00%, 28580/28580, 01:33:59/01:33:59, 5.07i/s] (3.4938595f0, 2.0767374f0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "S2S_v1(Embed(P(KnetArray{Float32,2}(512,38126))), LSTM(input=512,hidden=512,bidirectional,layers=2,dropout=0.2), Embed(P(KnetArray{Float32,2}(512,18857))), LSTM(input=512,hidden=512,layers=4,dropout=0.2), Linear(P(KnetArray{Float32,2}(18857,512)), P(KnetArray{Float32,1}(18857))), 0.2, Vocab(Dict(\"ağacından\" => 35370,\"komuta\" => 13566,\"ellisi\" => 25239,\"adresini\" => 22820,\"yüzeyi\" => 4051,\"paris'te\" => 9494,\"kafamdaki\" => 18790,\"yüzeyinde\" => 5042,\"geçerlidir\" => 6612,\"kökten\" => 7774…), [\"<s>\", \"<unk>\", \".\", \",\", \"bir\", \"ve\", \"bu\", \"''\", \"``\", \"için\"  …  \"seçmemiz\", \"destekleyip\", \"karşılaştırılabilir\", \"ördeğin\", \"gününüzü\", \"bağışçı\", \"istismara\", \"yaşça\", \"tedci\", \"fakültesi'nde\"], 2, 1, split), Vocab(Dict(\"middle-income\" => 13398,\"photosynthesis\" => 7689,\"polarizing\" => 17881,\"henry\" => 4248,\"abducted\" => 15691,\"rises\" => 6225,\"hampshire\" => 13888,\"whiz\" => 16835,\"cost-benefit\" => 13137,\"progression\" => 5549…), [\"<s>\", \"<unk>\", \",\", \".\", \"the\", \"and\", \"to\", \"of\", \"a\", \"that\"  …  \"archaea\", \"handshake\", \"brit\", \"wiper\", \"heroines\", \"coca\", \"exceptionally\", \"gallbladder\", \"autopsies\", \"linguistics\"], 2, 1, split))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bMDsmBYUsBW9",
        "colab_type": "text"
      },
      "source": [
        "### Save & Load"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Zbh36VesD1M",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "8c90a838-8fc7-4e1e-e035-dd5514845ddc"
      },
      "source": [
        "Knet.save(\"s2s_v1.jld2\",\"model\",model)\n",
        "model = Knet.load(\"s2s_v1.jld2\",\"model\")"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "S2S_v1(Embed(P(KnetArray{Float32,2}(512,38126))), LSTM(input=512,hidden=512,bidirectional,layers=2,dropout=0.2), Embed(P(KnetArray{Float32,2}(512,18857))), LSTM(input=512,hidden=512,layers=4,dropout=0.2), Linear(P(KnetArray{Float32,2}(18857,512)), P(KnetArray{Float32,1}(18857))), 0.2, Vocab(Dict(\"dev\" => 1277,\"komuta\" => 13566,\"ellisi\" => 25239,\"adresini\" => 22820,\"yüzeyi\" => 4051,\"paris'te\" => 9494,\"kafamdaki\" => 18790,\"yüzeyinde\" => 5042,\"geçerlidir\" => 6612,\"kökten\" => 7774…), [\"<s>\", \"<unk>\", \".\", \",\", \"bir\", \"ve\", \"bu\", \"''\", \"``\", \"için\"  …  \"seçmemiz\", \"destekleyip\", \"karşılaştırılabilir\", \"ördeğin\", \"gününüzü\", \"bağışçı\", \"istismara\", \"yaşça\", \"tedci\", \"fakültesi'nde\"], 2, 1, split), Vocab(Dict(\"middle-income\" => 13398,\"photosynthesis\" => 7689,\"polarizing\" => 17881,\"henry\" => 4248,\"abducted\" => 15691,\"rises\" => 6225,\"hampshire\" => 13888,\"whiz\" => 16835,\"cost-benefit\" => 13137,\"progression\" => 5549…), [\"<s>\", \"<unk>\", \",\", \".\", \"the\", \"and\", \"to\", \"of\", \"a\", \"that\"  …  \"archaea\", \"handshake\", \"brit\", \"wiper\", \"heroines\", \"coca\", \"exceptionally\", \"gallbladder\", \"autopsies\", \"linguistics\"], 2, 1, split))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ze5--Ke9rOu-",
        "colab_type": "text"
      },
      "source": [
        "### Translation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d8fIBofQpXhR",
        "colab_type": "code",
        "outputId": "1f85b982-26be-41c7-aa9d-f798709b7c24",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "function (s::S2S_v1)(src::Matrix{Int}; stopfactor = 3)\n",
        "    # Preperation for initial step\n",
        "    B,Tx = size(src)\n",
        "    max_step = stopfactor * Tx\n",
        "    tgt_eos = s.tgtvocab.eos\n",
        "\n",
        "    tgt = fill(tgt_eos, (B, 1)) # size as (B,2)\n",
        "    #output = fill(tgt_eos,(B,max_step))\n",
        "    output = Array{Int64}(undef, B, max_step)\n",
        "\n",
        "    rnn_encoder = s.encoder\n",
        "    rnn_decoder = s.decoder\n",
        "    project = s.projection\n",
        "\n",
        "    emb_out_src = s.srcembed(src)\n",
        "\n",
        "    # Safe for repetitive usage\n",
        "    rnn_encoder.h = 0\n",
        "    rnn_encoder.c = 0\n",
        "\n",
        "    y_enc = rnn_encoder(emb_out_src)\n",
        "    rnn_decoder.h = rnn_encoder.h\n",
        "    rnn_decoder.c = rnn_encoder.c\n",
        "\n",
        "    step = 0\n",
        "    #@test Ty == size(tgt,2)\n",
        "\n",
        "    while step < max_step\n",
        "        step += 1\n",
        "        emb_out_tgt = s.tgtembed(tgt)\n",
        "\n",
        "        y_dec = rnn_decoder(emb_out_tgt)\n",
        "\n",
        "        project_inp = reshape(y_dec, :, B)\n",
        "        project_out = project(project_inp)\n",
        "\n",
        "        eos_num = 0\n",
        "        for i = 1:B\n",
        "            # Assigns the position of the highest token\n",
        "            col = project_out[:, i]\n",
        "            colMax = col[1]\n",
        "            index = 1\n",
        "            for j in 1:length(col)\n",
        "                if colMax<col[j]\n",
        "                    colMax, index = col[j],j\n",
        "                end\n",
        "            end\n",
        "            if index == tgt_eos\n",
        "                eos_num += 1\n",
        "            end\n",
        "            tgt[i] = index\n",
        "        end\n",
        "\n",
        "        output[:,step] = tgt\n",
        "        eos_num == B && break # all produced eos\n",
        "\n",
        "\n",
        "    end\n",
        "\n",
        "    return output[:, 1:step]\n",
        "\n",
        "end\n",
        "\n",
        "\n",
        "## Utility to convert int arrays to sentence strings\n",
        "function int2str(y, vocab)\n",
        "    y = vec(y)\n",
        "    ysos = findnext(w -> !isequal(w, vocab.eos), y, 1)\n",
        "    ysos == nothing && return \"\"\n",
        "    yeos = something(findnext(isequal(vocab.eos), y, ysos), 1 + length(y))\n",
        "    join(vocab.i2w[y[ysos:yeos-1]], \" \")\n",
        "end"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "int2str (generic function with 1 method)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D1N5BGGy26Hs",
        "colab_type": "text"
      },
      "source": [
        "### Generate Translations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qf7RqMtd2_nB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "817db1b7-24f7-4902-a17e-9a34898f012a"
      },
      "source": [
        "@info \"Generating some translations\"\n",
        "d = MTData(tr_dev, en_dev, batchsize = 1) |> collect\n",
        "(src, tgt) = rand(d)\n",
        "out = model(src)\n",
        "println(\"SRC: \", int2str(src, model.srcvocab))\n",
        "println(\"REF: \", int2str(tgt, model.tgtvocab))\n",
        "println(\"OUT: \", int2str(out, model.tgtvocab))"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SRC: <unk> ilgilenmek yapabileceğiniz en tedavi edici ve meydan okuyucu davranış .\n",
            "REF: gardening is the most therapeutic and <unk> act you can do .\n",
            "OUT: the best way to do is to change the most effective and <unk> and <unk> .\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "┌ Info: Generating some translations\n",
            "└ @ Main In[33]:1\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2i7McH-4rm6W",
        "colab_type": "text"
      },
      "source": [
        "### Define BLEU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vchdolrErSIO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "68fd5523-56f9-4717-e2fb-687d68a2bbc7"
      },
      "source": [
        "CuArrays.usage_limit[] = 10_000_000_000\n",
        "\n",
        "function bleu(s2s, d::MTData)\n",
        "    d = MTData(d.src, d.tgt, batchsize = 1)\n",
        "    reffile = d.tgt.file\n",
        "    hypfile, hyp = mktemp()\n",
        "    for (x, y) in progress(collect(d))\n",
        "        g = s2s(x)\n",
        "        for i = 1:size(y, 1)\n",
        "            println(hyp, int2str(g[i, :], d.tgt.vocab))\n",
        "        end\n",
        "    end\n",
        "    close(hyp)\n",
        "    isfile(\"multi-bleu.perl\") || download(\n",
        "        \"https://github.com/moses-smt/mosesdecoder/raw/master/scripts/generic/multi-bleu.perl\",\n",
        "        \"multi-bleu.perl\",\n",
        "    )\n",
        "    run(pipeline(`cat $hypfile`, `perl multi-bleu.perl $reffile`))\n",
        "    return hypfile\n",
        "end\n",
        "\n",
        "# Calculating dev BLEU takes about 45 secs on a V100. We get about 8.0 BLEU which is pretty\n",
        "# low. As can be seen from the sample translations a loss of ~3+ (perplexity ~20+) or a BLEU\n",
        "# of ~8 is not sufficient to generate meaningful translations.\n",
        "\n"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "bleu (generic function with 1 method)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JtPfXeBZ5mJP",
        "colab_type": "text"
      },
      "source": [
        "### Calculate BLEU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_-TKoozM3kA5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 236
        },
        "outputId": "82cdab99-7c53-4709-870a-c8b48c6ee642"
      },
      "source": [
        "@info \"Calculating BLEU\"\n",
        "GC.gc();bleu(model, ddev)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "┌ Info: Calculating BLEU\n",
            "└ @ Main In[35]:1\n",
            "┣                    ┫ [0.35%, 14/4045, 00:59/04:42:21, 5.12s/i] "
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "InterruptException",
          "evalue": "ignored",
          "traceback": [
            "InterruptException:",
            "",
            "Stacktrace:",
            " [1] #call#29(::Int64, ::S2S_v1, ::Array{Int64,2}) at ./In[31]:44",
            " [2] S2S_v1 at ./In[31]:3 [inlined]",
            " [3] bleu(::S2S_v1, ::MTData) at ./In[34]:8",
            " [4] top-level scope at In[35]:2"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZyTjM1bA5vfN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}