Making_the_training_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Making the training data.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c0O4hH6thsGP"
      },
      "source": [
        "# Creating the Dataset\n",
        "\n",
        "credit : \n",
        "\n",
        "For the program to convert file in required format:\n",
        "\n",
        "https://github.com/magenta/magenta/tree/master/magenta/models/sketch_rnn\n",
        "\n",
        "For the method to load mnis data:\n",
        "\n",
        "https://medium.com/the-artificial-impostor/notes-understanding-tensorflow-part-2-f7e5ece849f5\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R1VZ5hVoKtK5",
        "outputId": "0b9f9692-6d6f-4c20-a746-5c843c0d0c20",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 717
        }
      },
      "source": [
        "!pip install numpy==1.16.2   \n",
        "!pip install tensorflow==1.14.0\n",
        "!pip install tensorflow-probability==0.7.0\n",
        "!pip install kfac==0.2.0\n",
        "!pip install keras==2.0.8 #Downgraded all files to be compatible with the code\n",
        "import numpy as np\n",
        "print(np.__version__)\n",
        "import time\n",
        "import random\n",
        "\n",
        "import codecs\n",
        "import collections\n",
        "import os\n",
        "import math\n",
        "import json\n",
        "import tensorflow as tf\n",
        "from six.moves import xrange"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: numpy==1.16.2 in /usr/local/lib/python3.6/dist-packages (1.16.2)\n",
            "Requirement already satisfied: tensorflow==1.14.0 in /usr/local/lib/python3.6/dist-packages (1.14.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (0.8.1)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (0.3.3)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (1.1.0)\n",
            "Requirement already satisfied: tensorboard<1.15.0,>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (1.14.0)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (0.2.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (1.0.8)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (1.32.0)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (3.12.4)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (0.35.1)\n",
            "Requirement already satisfied: tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (1.14.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (1.1.2)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (0.10.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (1.15.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (1.12.1)\n",
            "Requirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14.0) (1.16.2)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (50.3.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (3.2.2)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (1.0.1)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow==1.14.0) (2.10.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (2.0.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14.0) (3.2.0)\n",
            "Requirement already satisfied: tensorflow-probability==0.7.0 in /usr/local/lib/python3.6/dist-packages (0.7.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability==0.7.0) (1.15.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability==0.7.0) (4.4.2)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability==0.7.0) (1.16.2)\n",
            "Requirement already satisfied: cloudpickle>=0.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability==0.7.0) (1.3.0)\n",
            "Requirement already satisfied: kfac==0.2.0 in /usr/local/lib/python3.6/dist-packages (0.2.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from kfac==0.2.0) (1.15.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from kfac==0.2.0) (1.16.2)\n",
            "Requirement already satisfied: tensorflow-probability in /usr/local/lib/python3.6/dist-packages (from kfac==0.2.0) (0.7.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability->kfac==0.2.0) (4.4.2)\n",
            "Requirement already satisfied: cloudpickle>=0.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability->kfac==0.2.0) (1.3.0)\n",
            "Requirement already satisfied: keras==2.0.8 in /usr/local/lib/python3.6/dist-packages (2.0.8)\n",
            "Requirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.6/dist-packages (from keras==2.0.8) (1.16.2)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from keras==2.0.8) (1.15.0)\n",
            "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from keras==2.0.8) (1.4.1)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras==2.0.8) (3.13)\n",
            "1.16.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZjEgPfOqK_JH",
        "outputId": "dc19c8cd-e05a-4220-feb0-6ee01b8ee86c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2162
        }
      },
      "source": [
        "!pip install magenta\n",
        "import magenta\n",
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "import os\n",
        "import random\n",
        "from keras.layers.recurrent import LSTM"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: magenta in /usr/local/lib/python3.6/dist-packages (2.1.2)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.6/dist-packages (from magenta) (0.10.0)\n",
            "Requirement already satisfied: pygtrie>=2.3 in /usr/local/lib/python3.6/dist-packages (from magenta) (2.3.3)\n",
            "Requirement already satisfied: imageio in /usr/local/lib/python3.6/dist-packages (from magenta) (2.4.1)\n",
            "Requirement already satisfied: dm-sonnet in /usr/local/lib/python3.6/dist-packages (from magenta) (2.0.0)\n",
            "Requirement already satisfied: numba<0.50 in /usr/local/lib/python3.6/dist-packages (from magenta) (0.48.0)\n",
            "Requirement already satisfied: note-seq in /usr/local/lib/python3.6/dist-packages (from magenta) (0.0.1)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from magenta) (1.15.0)\n",
            "Requirement already satisfied: sk-video in /usr/local/lib/python3.6/dist-packages (from magenta) (1.1.10)\n",
            "Requirement already satisfied: matplotlib>=1.5.3 in /usr/local/lib/python3.6/dist-packages (from magenta) (3.2.2)\n",
            "Requirement already satisfied: pretty-midi>=0.2.6 in /usr/local/lib/python3.6/dist-packages (from magenta) (0.2.9)\n",
            "Requirement already satisfied: tensor2tensor in /usr/local/lib/python3.6/dist-packages (from magenta) (1.15.7)\n",
            "Requirement already satisfied: Pillow>=3.4.2 in /usr/local/lib/python3.6/dist-packages (from magenta) (7.0.0)\n",
            "Requirement already satisfied: tensorflow-probability in /usr/local/lib/python3.6/dist-packages (from magenta) (0.7.0)\n",
            "Requirement already satisfied: dopamine-rl<=3.0.1 in /usr/local/lib/python3.6/dist-packages (from magenta) (1.0.5)\n",
            "Requirement already satisfied: tensorflow in /usr/local/lib/python3.6/dist-packages (from magenta) (1.14.0)\n",
            "Requirement already satisfied: sox>=1.3.7 in /usr/local/lib/python3.6/dist-packages (from magenta) (1.4.1)\n",
            "Requirement already satisfied: python-rtmidi<1.2,>=1.1 in /usr/local/lib/python3.6/dist-packages (from magenta) (1.1.2)\n",
            "Requirement already satisfied: tensorflow-datasets in /usr/local/lib/python3.6/dist-packages (from magenta) (2.1.0)\n",
            "Requirement already satisfied: mir-eval>=0.4 in /usr/local/lib/python3.6/dist-packages (from magenta) (0.6)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from magenta) (1.4.1)\n",
            "Requirement already satisfied: mido==1.2.6 in /usr/local/lib/python3.6/dist-packages (from magenta) (1.2.6)\n",
            "Requirement already satisfied: scikit-image in /usr/local/lib/python3.6/dist-packages (from magenta) (0.16.2)\n",
            "Requirement already satisfied: tf-slim in /usr/local/lib/python3.6/dist-packages (from magenta) (1.1.0)\n",
            "Requirement already satisfied: librosa<0.8.0,>=0.6.2 in /usr/local/lib/python3.6/dist-packages (from magenta) (0.6.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from magenta) (1.16.2)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.6/dist-packages (from magenta) (0.35.1)\n",
            "Requirement already satisfied: tabulate>=0.7.5 in /usr/local/lib/python3.6/dist-packages (from dm-sonnet->magenta) (0.8.7)\n",
            "Requirement already satisfied: dm-tree>=0.1.1 in /usr/local/lib/python3.6/dist-packages (from dm-sonnet->magenta) (0.1.5)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from dm-sonnet->magenta) (1.12.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from numba<0.50->magenta) (50.3.0)\n",
            "Requirement already satisfied: llvmlite<0.32.0,>=0.31.0dev0 in /usr/local/lib/python3.6/dist-packages (from numba<0.50->magenta) (0.31.0)\n",
            "Requirement already satisfied: pandas>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from note-seq->magenta) (1.1.2)\n",
            "Requirement already satisfied: bokeh>=0.12.0 in /usr/local/lib/python3.6/dist-packages (from note-seq->magenta) (2.1.1)\n",
            "Requirement already satisfied: intervaltree>=2.1.0 in /usr/local/lib/python3.6/dist-packages (from note-seq->magenta) (2.1.0)\n",
            "Requirement already satisfied: attrs in /usr/local/lib/python3.6/dist-packages (from note-seq->magenta) (20.2.0)\n",
            "Requirement already satisfied: IPython in /usr/local/lib/python3.6/dist-packages (from note-seq->magenta) (5.5.0)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from note-seq->magenta) (3.12.4)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=1.5.3->magenta) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=1.5.3->magenta) (1.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=1.5.3->magenta) (2.8.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=1.5.3->magenta) (2.4.7)\n",
            "Requirement already satisfied: gin-config in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (0.3.0)\n",
            "Requirement already satisfied: google-api-python-client in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (1.7.12)\n",
            "Requirement already satisfied: bz2file in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (0.98)\n",
            "Requirement already satisfied: gym in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (0.17.2)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (4.1.2.30)\n",
            "Requirement already satisfied: tensorflow-gan in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (2.0.0)\n",
            "Requirement already satisfied: gevent in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (20.9.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (0.16.0)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (2.10.0)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (1.1.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (4.41.1)\n",
            "Requirement already satisfied: oauth2client in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (4.1.3)\n",
            "Requirement already satisfied: mesh-tensorflow in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (0.1.16)\n",
            "Requirement already satisfied: pypng in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (0.0.20)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (2.23.0)\n",
            "Requirement already satisfied: gunicorn in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (20.0.4)\n",
            "Requirement already satisfied: kfac in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (0.2.0)\n",
            "Requirement already satisfied: tensorflow-addons in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (0.8.3)\n",
            "Requirement already satisfied: flask in /usr/local/lib/python3.6/dist-packages (from tensor2tensor->magenta) (1.1.2)\n",
            "Requirement already satisfied: cloudpickle>=0.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability->magenta) (1.3.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from tensorflow-probability->magenta) (4.4.2)\n",
            "Requirement already satisfied: tensorflow-estimator<1.15.0rc0,>=1.14.0rc0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->magenta) (1.14.0)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow->magenta) (1.32.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->magenta) (0.8.1)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow->magenta) (1.1.2)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->magenta) (1.1.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow->magenta) (1.0.8)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow->magenta) (0.2.0)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->magenta) (0.3.3)\n",
            "Requirement already satisfied: tensorboard<1.15.0,>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->magenta) (1.14.0)\n",
            "Requirement already satisfied: tensorflow-metadata in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->magenta) (0.24.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->magenta) (0.3.2)\n",
            "Requirement already satisfied: promise in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->magenta) (2.3)\n",
            "Requirement already satisfied: networkx>=2.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image->magenta) (2.5)\n",
            "Requirement already satisfied: PyWavelets>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from scikit-image->magenta) (1.1.1)\n",
            "Requirement already satisfied: audioread>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from librosa<0.8.0,>=0.6.2->magenta) (2.1.8)\n",
            "Requirement already satisfied: scikit-learn!=0.19.0,>=0.14.0 in /usr/local/lib/python3.6/dist-packages (from librosa<0.8.0,>=0.6.2->magenta) (0.22.2.post1)\n",
            "Requirement already satisfied: joblib>=0.12 in /usr/local/lib/python3.6/dist-packages (from librosa<0.8.0,>=0.6.2->magenta) (0.16.0)\n",
            "Requirement already satisfied: resampy>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from librosa<0.8.0,>=0.6.2->magenta) (0.2.2)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.18.1->note-seq->magenta) (2018.9)\n",
            "Requirement already satisfied: Jinja2>=2.7 in /usr/local/lib/python3.6/dist-packages (from bokeh>=0.12.0->note-seq->magenta) (2.11.2)\n",
            "Requirement already satisfied: tornado>=5.1 in /usr/local/lib/python3.6/dist-packages (from bokeh>=0.12.0->note-seq->magenta) (5.1.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.6/dist-packages (from bokeh>=0.12.0->note-seq->magenta) (3.7.4.3)\n",
            "Requirement already satisfied: PyYAML>=3.10 in /usr/local/lib/python3.6/dist-packages (from bokeh>=0.12.0->note-seq->magenta) (3.13)\n",
            "Requirement already satisfied: packaging>=16.8 in /usr/local/lib/python3.6/dist-packages (from bokeh>=0.12.0->note-seq->magenta) (20.4)\n",
            "Requirement already satisfied: sortedcontainers in /usr/local/lib/python3.6/dist-packages (from intervaltree>=2.1.0->note-seq->magenta) (2.2.2)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.6/dist-packages (from IPython->note-seq->magenta) (1.0.18)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.6/dist-packages (from IPython->note-seq->magenta) (4.3.3)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from IPython->note-seq->magenta) (0.7.5)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from IPython->note-seq->magenta) (4.8.0)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.6/dist-packages (from IPython->note-seq->magenta) (0.8.1)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from IPython->note-seq->magenta) (2.6.1)\n",
            "Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client->tensor2tensor->magenta) (3.0.1)\n",
            "Requirement already satisfied: httplib2<1dev,>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client->tensor2tensor->magenta) (0.17.4)\n",
            "Requirement already satisfied: google-auth>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client->tensor2tensor->magenta) (1.17.2)\n",
            "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client->tensor2tensor->magenta) (0.0.4)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym->tensor2tensor->magenta) (1.5.0)\n",
            "Requirement already satisfied: tensorflow-hub>=0.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gan->tensor2tensor->magenta) (0.9.0)\n",
            "Requirement already satisfied: zope.interface in /usr/local/lib/python3.6/dist-packages (from gevent->tensor2tensor->magenta) (5.1.2)\n",
            "Requirement already satisfied: zope.event in /usr/local/lib/python3.6/dist-packages (from gevent->tensor2tensor->magenta) (4.5.0)\n",
            "Requirement already satisfied: greenlet>=0.4.17; platform_python_implementation == \"CPython\" in /usr/local/lib/python3.6/dist-packages (from gevent->tensor2tensor->magenta) (0.4.17)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.6/dist-packages (from sympy->tensor2tensor->magenta) (1.1.0)\n",
            "Requirement already satisfied: pyasn1>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from oauth2client->tensor2tensor->magenta) (0.4.8)\n",
            "Requirement already satisfied: rsa>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from oauth2client->tensor2tensor->magenta) (4.6)\n",
            "Requirement already satisfied: pyasn1-modules>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from oauth2client->tensor2tensor->magenta) (0.2.8)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->tensor2tensor->magenta) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->tensor2tensor->magenta) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->tensor2tensor->magenta) (2020.6.20)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->tensor2tensor->magenta) (1.24.3)\n",
            "Requirement already satisfied: typeguard in /usr/local/lib/python3.6/dist-packages (from tensorflow-addons->tensor2tensor->magenta) (2.7.1)\n",
            "Requirement already satisfied: Werkzeug>=0.15 in /usr/local/lib/python3.6/dist-packages (from flask->tensor2tensor->magenta) (1.0.1)\n",
            "Requirement already satisfied: itsdangerous>=0.24 in /usr/local/lib/python3.6/dist-packages (from flask->tensor2tensor->magenta) (1.1.0)\n",
            "Requirement already satisfied: click>=5.1 in /usr/local/lib/python3.6/dist-packages (from flask->tensor2tensor->magenta) (7.1.2)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow->magenta) (3.2.2)\n",
            "Requirement already satisfied: googleapis-common-protos<2,>=1.52.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-metadata->tensorflow-datasets->magenta) (1.52.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.7->bokeh>=0.12.0->note-seq->magenta) (1.1.1)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->IPython->note-seq->magenta) (0.2.5)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.6/dist-packages (from traitlets>=4.2->IPython->note-seq->magenta) (0.2.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->IPython->note-seq->magenta) (0.6.0)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.4.1->google-api-python-client->tensor2tensor->magenta) (4.1.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow->magenta) (2.0.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow->magenta) (3.2.0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7pOtzYM2LXJq",
        "outputId": "ed4e93cc-bfa0-4931-ebe2-834a907782f4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        }
      },
      "source": [
        "mnist = input_data.read_data_sets(os.getcwd() + \"/MNIST-data/\", one_hot=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-3-80da94044662>:1: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please write your own downloading logic.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting /content/MNIST-data/train-images-idx3-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.data to implement this functionality.\n",
            "Extracting /content/MNIST-data/train-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use tf.one_hot on tensors.\n",
            "Extracting /content/MNIST-data/t10k-images-idx3-ubyte.gz\n",
            "Extracting /content/MNIST-data/t10k-labels-idx1-ubyte.gz\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I68skeM4Le_z",
        "outputId": "b32d2e8c-c5d4-48f2-903c-435b2fab4bc6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline \n",
        "\n",
        "\n",
        "old_v = tf.logging.get_verbosity()\n",
        "tf.logging.set_verbosity(tf.logging.ERROR)\n",
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "mnist = input_data.read_data_sets(\"/home/meysam/github/ML-with-TensorFlow/Datasets/mnist\", one_hot=True)\n",
        "\n",
        "# Load data\n",
        "train_data =  np.append(mnist.train.images,mnist.train.labels)\n",
        "validation_data =  np.append(mnist.validation.images,mnist.validation.labels)\n",
        "test_data=np.append(mnist.test.images,mnist.test.labels)\n",
        "train_data1=mnist.train.images\n",
        "validation_data1=mnist.validation.images\n",
        "test_data1=mnist.test.images\n",
        "X_train = mnist.train.images.reshape([-1,28,28])\n",
        "Y_train = mnist.train.labels\n",
        "X_validation = mnist.validation.images.reshape([-1,28,28])\n",
        "Y_validation = mnist.validation.labels\n",
        "X_test = mnist.test.images.reshape([-1,28,28])\n",
        "Y_test = mnist.test.labels\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Extracting /home/meysam/github/ML-with-TensorFlow/Datasets/mnist/train-images-idx3-ubyte.gz\n",
            "Extracting /home/meysam/github/ML-with-TensorFlow/Datasets/mnist/train-labels-idx1-ubyte.gz\n",
            "Extracting /home/meysam/github/ML-with-TensorFlow/Datasets/mnist/t10k-images-idx3-ubyte.gz\n",
            "Extracting /home/meysam/github/ML-with-TensorFlow/Datasets/mnist/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3I4OB-5qimIy"
      },
      "source": [
        "# Testing out various combinations of the numpy arrays and seeing which is the best fit for training our model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bCrV3ZDSLhXG",
        "outputId": "ae68c86a-0073-4921-aa4b-56cd9d08c64d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        }
      },
      "source": [
        "print('Check the dimenssions')\n",
        "print('X_train.shape:', X_train.shape)\n",
        "print('Y_train.shape:', Y_train.shape)\n",
        "print('X_validation.shape:', X_validation.shape)\n",
        "print('X_validation.shape:', X_validation.shape)\n",
        "print('X_test.shape:', X_test.shape)\n",
        "print('Y_test.shape:', Y_test.shape)\n",
        "print('train_data.shape:', train_data.shape)\n",
        "print('train_data1.shape:', train_data1.shape)\n",
        "print('validation_data.shape:', validation_data.shape)\n",
        "print('validation_data1.shape:', validation_data1.shape)\n",
        "print('test_data.shape:', test_data.shape)\n",
        "print('test_data1.shape:', test_data1.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Check the dimenssions\n",
            "X_train.shape: (55000, 28, 28)\n",
            "Y_train.shape: (55000, 10)\n",
            "X_validation.shape: (5000, 28, 28)\n",
            "X_validation.shape: (5000, 28, 28)\n",
            "X_test.shape: (10000, 28, 28)\n",
            "Y_test.shape: (10000, 10)\n",
            "train_data.shape: (43670000,)\n",
            "train_data1.shape: (55000, 784)\n",
            "validation_data.shape: (3970000,)\n",
            "validation_data1.shape: (5000, 784)\n",
            "test_data.shape: (7940000,)\n",
            "test_data1.shape: (10000, 784)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2yi748t_nj6F",
        "outputId": "e8304422-9dda-4925-c7e9-cc8c1a9110e0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        }
      },
      "source": [
        "reshapetostroke_train = mnist.train.images.reshape([28,28,-1])\n",
        "print(reshapetostroke_train.shape)\n",
        "reshapetostroke_valid = mnist.validation.images.reshape([28,28,-1])\n",
        "print(reshapetostroke_valid.shape)\n",
        "reshapetostroke_test = mnist.validation.images.reshape([28,28,-1])\n",
        "print(reshapetostroke_test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(28, 28, 55000)\n",
            "(28, 28, 5000)\n",
            "(28, 28, 5000)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ncZB5aAliyr0"
      },
      "source": [
        "# Saving the file in the correct format"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8nom8tprLiac"
      },
      "source": [
        "filename = os.path.join('/content', 'mnist_lstmv6.npz')\n",
        "np.savez_compressed(filename, train=reshapetostroke_train, valid=reshapetostroke_valid, test=reshapetostroke_test)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}