590_model_training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-AVPUxuiELZ7"
   },
   "source": [
    "## 590 Project：Doodle Image Recognition Using LSTM+CNN model\n",
    "## Yarou Xu, Fengdi Li, Yifan Wu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Set up working environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 280
    },
    "colab_type": "code",
    "id": "xOg2TCc39EQW",
    "outputId": "d8080917-a599-487e-df09-b614462fb07c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting keras==2.1.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bf/c2/b0c2ece713e754d1692aa432ad682751cd1ad6abf7500a534558b1fbfbe7/Keras-2.1.0-py2.py3-none-any.whl (302kB)\n",
      "\u001b[K    100% |████████████████████████████████| 307kB 9.2MB/s \n",
      "\u001b[?25hRequirement already satisfied: numpy>=1.9.1 in /usr/local/lib/python3.6/dist-packages (from keras==2.1.0) (1.14.6)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from keras==2.1.0) (3.13)\n",
      "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/dist-packages (from keras==2.1.0) (1.11.0)\n",
      "Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from keras==2.1.0) (1.1.0)\n",
      "\u001b[31mtextgenrnn 1.4.1 has requirement keras>=2.1.5, but you'll have keras 2.1.0 which is incompatible.\u001b[0m\n",
      "Installing collected packages: keras\n",
      "  Found existing installation: Keras 2.2.4\n",
      "    Uninstalling Keras-2.2.4:\n",
      "      Successfully uninstalled Keras-2.2.4\n",
      "Successfully installed keras-2.1.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'2.2.4'"
      ]
     },
     "execution_count": 3,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!pip install  keras==2.1.0\n",
    "import keras\n",
    "keras.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 263
    },
    "colab_type": "code",
    "id": "bPZ02CbW9HUv",
    "outputId": "91898ba6-e77b-44e6-f1cb-befdf7227a00"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gputil\n",
      "  Downloading https://files.pythonhosted.org/packages/45/99/837428d26b47ebd6b66d6e1b180e98ec4a557767a93a81a02ea9d6242611/GPUtil-1.3.0.tar.gz\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from gputil) (1.14.6)\n",
      "Building wheels for collected packages: gputil\n",
      "  Running setup.py bdist_wheel for gputil ... \u001b[?25l-\b \bdone\n",
      "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/17/0f/04/b79c006972335e35472c0b835ed52bfc0815258d409f560108\n",
      "Successfully built gputil\n",
      "Installing collected packages: gputil\n",
      "Successfully installed gputil-1.3.0\n",
      "Requirement already satisfied: psutil in /usr/local/lib/python3.6/dist-packages (5.4.8)\n",
      "Requirement already satisfied: humanize in /usr/local/lib/python3.6/dist-packages (0.5.1)\n",
      "Gen RAM Free: 12.9 GB  | Proc size: 324.2 MB\n",
      "GPU RAM Free: 11441MB | Used: 0MB | Util   0% | Total 11441MB\n"
     ]
    }
   ],
   "source": [
    "!ln -sf /opt/bin/nvidia-smi /usr/bin/nvidia-smi\n",
    "!pip install gputil\n",
    "!pip install psutil\n",
    "!pip install humanize\n",
    "import psutil\n",
    "import humanize\n",
    "import os\n",
    "import GPUtil as GPU\n",
    "GPUs = GPU.getGPUs()\n",
    "# XXX: only one GPU on Colab and isn’t guaranteed\n",
    "gpu = GPUs[0]\n",
    "def printm():\n",
    "  process = psutil.Process(os.getpid())\n",
    "  print(\"Gen RAM Free: \" + humanize.naturalsize( psutil.virtual_memory().available ), \" | Proc size: \" + humanize.naturalsize( process.memory_info().rss))\n",
    "  print(\"GPU RAM Free: {0:.0f}MB | Used: {1:.0f}MB | Util {2:3.0f}% | Total {3:.0f}MB\".format(gpu.memoryFree, gpu.memoryUsed, gpu.memoryUtil*100, gpu.memoryTotal))\n",
    "printm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 124
    },
    "colab_type": "code",
    "id": "q4aQadSU9Jwx",
    "outputId": "751bb3ab-0d17-4d74-ca29-1e282cff931f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZYjiRvA5Ed2U"
   },
   "source": [
    "## 2. Preprocessing the quick draw data:\n",
    "     a. Filter sketching drawn by Chinese and American.\n",
    "     b. Filter sketchings that has been successfully recognized."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 5911
    },
    "colab_type": "code",
    "id": "dZQo2JaMBeIM",
    "outputId": "72485b8e-1f34-42a7-dc17-669fdd9f3e32"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bread.csv',\n",
       " 'pliers.csv',\n",
       " 'remote control.csv',\n",
       " 'pickup truck.csv',\n",
       " 'envelope.csv',\n",
       " 'popsicle.csv',\n",
       " 'sea turtle.csv',\n",
       " 'elephant.csv',\n",
       " 'castle.csv',\n",
       " 'ambulance.csv',\n",
       " 'river.csv',\n",
       " 'bandage.csv',\n",
       " 'nail.csv',\n",
       " 'line.csv',\n",
       " 'bucket.csv',\n",
       " 'bus.csv',\n",
       " 'cello.csv',\n",
       " 'ocean.csv',\n",
       " 'truck.csv',\n",
       " 'camouflage.csv',\n",
       " 'harp.csv',\n",
       " 'telephone.csv',\n",
       " 'stairs.csv',\n",
       " 'star.csv',\n",
       " 'guitar.csv',\n",
       " 'sandwich.csv',\n",
       " 'sun.csv',\n",
       " 'feather.csv',\n",
       " 'leaf.csv',\n",
       " 'toilet.csv',\n",
       " 'strawberry.csv',\n",
       " 'waterslide.csv',\n",
       " 'bottlecap.csv',\n",
       " 'coffee cup.csv',\n",
       " 'banana.csv',\n",
       " 'dresser.csv',\n",
       " 'house plant.csv',\n",
       " 'skateboard.csv',\n",
       " 'skyscraper.csv',\n",
       " 'pizza.csv',\n",
       " 'hammer.csv',\n",
       " 'teapot.csv',\n",
       " 'giraffe.csv',\n",
       " 'underwear.csv',\n",
       " 'snowman.csv',\n",
       " 'monkey.csv',\n",
       " 'computer.csv',\n",
       " 'pencil.csv',\n",
       " 'shovel.csv',\n",
       " 'necklace.csv',\n",
       " 'compass.csv',\n",
       " 'bat.csv',\n",
       " 'bicycle.csv',\n",
       " 'teddy-bear.csv',\n",
       " 'scorpion.csv',\n",
       " 'hot dog.csv',\n",
       " 'fish.csv',\n",
       " 'see saw.csv',\n",
       " 'rain.csv',\n",
       " 'snail.csv',\n",
       " 'sink.csv',\n",
       " 'belt.csv',\n",
       " 'speedboat.csv',\n",
       " 'pants.csv',\n",
       " 'trombone.csv',\n",
       " 'crocodile.csv',\n",
       " 'broccoli.csv',\n",
       " 'hedgehog.csv',\n",
       " 'rainbow.csv',\n",
       " 'fork.csv',\n",
       " 'bulldozer.csv',\n",
       " 'sock.csv',\n",
       " 'snake.csv',\n",
       " 'paper clip.csv',\n",
       " 'bear.csv',\n",
       " 'marker.csv',\n",
       " 'birthday cake.csv',\n",
       " 'saxophone.csv',\n",
       " 'rake.csv',\n",
       " 'stethoscope.csv',\n",
       " 'broom.csv',\n",
       " 'crown.csv',\n",
       " 'square.csv',\n",
       " 'fire hydrant.csv',\n",
       " 'jail.csv',\n",
       " 'donut.csv',\n",
       " 'oven.csv',\n",
       " 'beard.csv',\n",
       " 'yoga.csv',\n",
       " 'The Eiffel Tower.csv',\n",
       " 'camera.csv',\n",
       " 'purse.csv',\n",
       " 'ice cream.csv',\n",
       " 'pig.csv',\n",
       " 'trumpet.csv',\n",
       " 'table.csv',\n",
       " 'bush.csv',\n",
       " 'rollerskates.csv',\n",
       " 'goatee.csv',\n",
       " 'cup.csv',\n",
       " 'anvil.csv',\n",
       " 'suitcase.csv',\n",
       " 'chair.csv',\n",
       " 'drill.csv',\n",
       " 'peanut.csv',\n",
       " 'squirrel.csv',\n",
       " 'matches.csv',\n",
       " 'sword.csv',\n",
       " 'cat.csv',\n",
       " 'toe.csv',\n",
       " 'snorkel.csv',\n",
       " 'pond.csv',\n",
       " 'calculator.csv',\n",
       " 'airplane.csv',\n",
       " 'squiggle.csv',\n",
       " 'blackberry.csv',\n",
       " 'ear.csv',\n",
       " 'frying pan.csv',\n",
       " 'chandelier.csv',\n",
       " 'lollipop.csv',\n",
       " 'binoculars.csv',\n",
       " 'garden.csv',\n",
       " 'basket.csv',\n",
       " 'penguin.csv',\n",
       " 'washing machine.csv',\n",
       " 'canoe.csv',\n",
       " 'screwdriver.csv',\n",
       " 'beach.csv',\n",
       " 'eyeglasses.csv',\n",
       " 'mouse.csv',\n",
       " 'apple.csv',\n",
       " 'van.csv',\n",
       " 'grapes.csv',\n",
       " 'grass.csv',\n",
       " 'watermelon.csv',\n",
       " 'floor lamp.csv',\n",
       " 'moon.csv',\n",
       " 'zigzag.csv',\n",
       " 'leg.csv',\n",
       " 'smiley face.csv',\n",
       " 'octagon.csv',\n",
       " 'dumbbell.csv',\n",
       " 'sweater.csv',\n",
       " 'stitches.csv',\n",
       " 'tractor.csv',\n",
       " 'foot.csv',\n",
       " 'helmet.csv',\n",
       " 'basketball.csv',\n",
       " 'crab.csv',\n",
       " 'clock.csv',\n",
       " 'diamond.csv',\n",
       " 'car.csv',\n",
       " 'axe.csv',\n",
       " 'traffic light.csv',\n",
       " 'sleeping bag.csv',\n",
       " 'baseball.csv',\n",
       " 'eye.csv',\n",
       " 'flower.csv',\n",
       " 'hot air balloon.csv',\n",
       " 'tree.csv',\n",
       " 'wine bottle.csv',\n",
       " 'hot tub.csv',\n",
       " 'peas.csv',\n",
       " 'door.csv',\n",
       " 'calendar.csv',\n",
       " 'wine glass.csv',\n",
       " 'stove.csv',\n",
       " 'hockey stick.csv',\n",
       " 'toothpaste.csv',\n",
       " 'moustache.csv',\n",
       " 'mountain.csv',\n",
       " 'tooth.csv',\n",
       " 'cannon.csv',\n",
       " 'firetruck.csv',\n",
       " 'shorts.csv',\n",
       " 'stereo.csv',\n",
       " 'cloud.csv',\n",
       " 'paintbrush.csv',\n",
       " 'pear.csv',\n",
       " 'dishwasher.csv',\n",
       " 'laptop.csv',\n",
       " 'frog.csv',\n",
       " 'vase.csv',\n",
       " 'diving board.csv',\n",
       " 'backpack.csv',\n",
       " 'lobster.csv',\n",
       " 'golf club.csv',\n",
       " 'garden hose.csv',\n",
       " 'hexagon.csv',\n",
       " 'bird.csv',\n",
       " 'finger.csv',\n",
       " 'animal migration.csv',\n",
       " 'steak.csv',\n",
       " 'mailbox.csv',\n",
       " 'shark.csv',\n",
       " 'television.csv',\n",
       " 'mermaid.csv',\n",
       " 'cow.csv',\n",
       " 'crayon.csv',\n",
       " 'palm tree.csv',\n",
       " 'windmill.csv',\n",
       " 'cookie.csv',\n",
       " 'kangaroo.csv',\n",
       " 'blueberry.csv',\n",
       " 'tiger.csv',\n",
       " 'tennis racquet.csv',\n",
       " 'dragon.csv',\n",
       " 'cell phone.csv',\n",
       " 'pineapple.csv',\n",
       " 'candle.csv',\n",
       " 'sheep.csv',\n",
       " 'cactus.csv',\n",
       " 'angel.csv',\n",
       " 'mosquito.csv',\n",
       " 'church.csv',\n",
       " 'couch.csv',\n",
       " 'The Great Wall of China.csv',\n",
       " 'tornado.csv',\n",
       " 'jacket.csv',\n",
       " 'nose.csv',\n",
       " 'octopus.csv',\n",
       " 'motorbike.csv',\n",
       " 'bracelet.csv',\n",
       " 'brain.csv',\n",
       " 'The Mona Lisa.csv',\n",
       " 'toothbrush.csv',\n",
       " 'carrot.csv',\n",
       " 'barn.csv',\n",
       " 'microphone.csv',\n",
       " 'zebra.csv',\n",
       " 'map.csv',\n",
       " 'camel.csv',\n",
       " 'wheel.csv',\n",
       " 'bridge.csv',\n",
       " 'lighthouse.csv',\n",
       " 'spreadsheet.csv',\n",
       " 'hockey puck.csv',\n",
       " 'wristwatch.csv',\n",
       " 'helicopter.csv',\n",
       " 'swan.csv',\n",
       " 'flamingo.csv',\n",
       " 'eraser.csv',\n",
       " 'bee.csv',\n",
       " 'flashlight.csv',\n",
       " 'megaphone.csv',\n",
       " 'ladder.csv',\n",
       " 'shoe.csv',\n",
       " 'asparagus.csv',\n",
       " 't-shirt.csv',\n",
       " 'passport.csv',\n",
       " 'hand.csv',\n",
       " 'triangle.csv',\n",
       " 'lightning.csv',\n",
       " 'mug.csv',\n",
       " 'submarine.csv',\n",
       " 'violin.csv',\n",
       " 'owl.csv',\n",
       " 'scissors.csv',\n",
       " 'baseball bat.csv',\n",
       " 'string bean.csv',\n",
       " 'lantern.csv',\n",
       " 'house.csv',\n",
       " 'elbow.csv',\n",
       " 'power outlet.csv',\n",
       " 'stop sign.csv',\n",
       " 'bed.csv',\n",
       " 'school bus.csv',\n",
       " 'hamburger.csv',\n",
       " 'lipstick.csv',\n",
       " 'light bulb.csv',\n",
       " 'flip flops.csv',\n",
       " 'alarm clock.csv',\n",
       " 'ant.csv',\n",
       " 'face.csv',\n",
       " 'microwave.csv',\n",
       " 'hourglass.csv',\n",
       " 'panda.csv',\n",
       " 'pool.csv',\n",
       " 'circle.csv',\n",
       " 'onion.csv',\n",
       " 'raccoon.csv',\n",
       " 'bowtie.csv',\n",
       " 'umbrella.csv',\n",
       " 'butterfly.csv',\n",
       " 'fireplace.csv',\n",
       " 'skull.csv',\n",
       " 'train.csv',\n",
       " 'mouth.csv',\n",
       " 'hat.csv',\n",
       " 'drums.csv',\n",
       " 'book.csv',\n",
       " 'radio.csv',\n",
       " 'roller coaster.csv',\n",
       " 'snowflake.csv',\n",
       " 'piano.csv',\n",
       " 'rhinoceros.csv',\n",
       " 'cake.csv',\n",
       " 'toaster.csv',\n",
       " 'paint can.csv',\n",
       " 'knee.csv',\n",
       " 'spider.csv',\n",
       " 'tent.csv',\n",
       " 'rabbit.csv',\n",
       " 'clarinet.csv',\n",
       " 'whale.csv',\n",
       " 'boomerang.csv',\n",
       " 'hospital.csv',\n",
       " 'ceiling fan.csv',\n",
       " 'saw.csv',\n",
       " 'pillow.csv',\n",
       " 'fence.csv',\n",
       " 'dog.csv',\n",
       " 'duck.csv',\n",
       " 'parrot.csv',\n",
       " 'swing set.csv',\n",
       " 'spoon.csv',\n",
       " 'fan.csv',\n",
       " 'cruise ship.csv',\n",
       " 'picture frame.csv',\n",
       " 'mushroom.csv',\n",
       " 'headphones.csv',\n",
       " 'horse.csv',\n",
       " 'flying saucer.csv',\n",
       " 'lion.csv',\n",
       " 'postcard.csv',\n",
       " 'bench.csv',\n",
       " 'keyboard.csv',\n",
       " 'parachute.csv',\n",
       " 'streetlight.csv',\n",
       " 'arm.csv',\n",
       " 'police car.csv',\n",
       " 'sailboat.csv',\n",
       " 'cooler.csv',\n",
       " 'bathtub.csv',\n",
       " 'campfire.csv',\n",
       " 'hurricane.csv',\n",
       " 'soccer ball.csv',\n",
       " 'potato.csv',\n",
       " 'dolphin.csv',\n",
       " 'key.csv']"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "# Check all the sketching categories\n",
    "path = '/content/drive/My Drive/quick_draw_training/'\n",
    "files = os.listdir(path)\n",
    "files\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "EyAIJ-UgCXA_",
    "outputId": "df9f58b4-843b-4568-ee5a-9e449735fa01"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "340"
      ]
     },
     "execution_count": 40,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(files)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QmJbOmimFYa9"
   },
   "source": [
    "Because the US sketching data is too big for the google colab to process, we select the top 100 and top 1000 images per category."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 5893
    },
    "colab_type": "code",
    "id": "JUh5f9qXCAKL",
    "outputId": "a234e5a2-a5d1-4a23-8bea-8796736eea93"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pliers.csv finished, 339 to go.\n",
      "remote control.csv finished, 338 to go.\n",
      "pickup truck.csv finished, 337 to go.\n",
      "envelope.csv finished, 336 to go.\n",
      "popsicle.csv finished, 335 to go.\n",
      "sea turtle.csv finished, 334 to go.\n",
      "elephant.csv finished, 333 to go.\n",
      "castle.csv finished, 332 to go.\n",
      "ambulance.csv finished, 331 to go.\n",
      "river.csv finished, 330 to go.\n",
      "bandage.csv finished, 329 to go.\n",
      "nail.csv finished, 328 to go.\n",
      "line.csv finished, 327 to go.\n",
      "bucket.csv finished, 326 to go.\n",
      "bus.csv finished, 325 to go.\n",
      "cello.csv finished, 324 to go.\n",
      "ocean.csv finished, 323 to go.\n",
      "truck.csv finished, 322 to go.\n",
      "camouflage.csv finished, 321 to go.\n",
      "harp.csv finished, 320 to go.\n",
      "telephone.csv finished, 319 to go.\n",
      "stairs.csv finished, 318 to go.\n",
      "star.csv finished, 317 to go.\n",
      "guitar.csv finished, 316 to go.\n",
      "sandwich.csv finished, 315 to go.\n",
      "sun.csv finished, 314 to go.\n",
      "feather.csv finished, 313 to go.\n",
      "leaf.csv finished, 312 to go.\n",
      "toilet.csv finished, 311 to go.\n",
      "strawberry.csv finished, 310 to go.\n",
      "waterslide.csv finished, 309 to go.\n",
      "bottlecap.csv finished, 308 to go.\n",
      "coffee cup.csv finished, 307 to go.\n",
      "banana.csv finished, 306 to go.\n",
      "dresser.csv finished, 305 to go.\n",
      "house plant.csv finished, 304 to go.\n",
      "skateboard.csv finished, 303 to go.\n",
      "skyscraper.csv finished, 302 to go.\n",
      "pizza.csv finished, 301 to go.\n",
      "hammer.csv finished, 300 to go.\n",
      "teapot.csv finished, 299 to go.\n",
      "giraffe.csv finished, 298 to go.\n",
      "underwear.csv finished, 297 to go.\n",
      "snowman.csv finished, 296 to go.\n",
      "monkey.csv finished, 295 to go.\n",
      "computer.csv finished, 294 to go.\n",
      "pencil.csv finished, 293 to go.\n",
      "shovel.csv finished, 292 to go.\n",
      "necklace.csv finished, 291 to go.\n",
      "compass.csv finished, 290 to go.\n",
      "bat.csv finished, 289 to go.\n",
      "bicycle.csv finished, 288 to go.\n",
      "teddy-bear.csv finished, 287 to go.\n",
      "scorpion.csv finished, 286 to go.\n",
      "hot dog.csv finished, 285 to go.\n",
      "fish.csv finished, 284 to go.\n",
      "see saw.csv finished, 283 to go.\n",
      "rain.csv finished, 282 to go.\n",
      "snail.csv finished, 281 to go.\n",
      "sink.csv finished, 280 to go.\n",
      "belt.csv finished, 279 to go.\n",
      "speedboat.csv finished, 278 to go.\n",
      "pants.csv finished, 277 to go.\n",
      "trombone.csv finished, 276 to go.\n",
      "crocodile.csv finished, 275 to go.\n",
      "broccoli.csv finished, 274 to go.\n",
      "hedgehog.csv finished, 273 to go.\n",
      "rainbow.csv finished, 272 to go.\n",
      "fork.csv finished, 271 to go.\n",
      "bulldozer.csv finished, 270 to go.\n",
      "sock.csv finished, 269 to go.\n",
      "snake.csv finished, 268 to go.\n",
      "paper clip.csv finished, 267 to go.\n",
      "bear.csv finished, 266 to go.\n",
      "marker.csv finished, 265 to go.\n",
      "birthday cake.csv finished, 264 to go.\n",
      "saxophone.csv finished, 263 to go.\n",
      "rake.csv finished, 262 to go.\n",
      "stethoscope.csv finished, 261 to go.\n",
      "broom.csv finished, 260 to go.\n",
      "crown.csv finished, 259 to go.\n",
      "square.csv finished, 258 to go.\n",
      "fire hydrant.csv finished, 257 to go.\n",
      "jail.csv finished, 256 to go.\n",
      "donut.csv finished, 255 to go.\n",
      "oven.csv finished, 254 to go.\n",
      "beard.csv finished, 253 to go.\n",
      "yoga.csv finished, 252 to go.\n",
      "The Eiffel Tower.csv finished, 251 to go.\n",
      "camera.csv finished, 250 to go.\n",
      "purse.csv finished, 249 to go.\n",
      "ice cream.csv finished, 248 to go.\n",
      "pig.csv finished, 247 to go.\n",
      "trumpet.csv finished, 246 to go.\n",
      "table.csv finished, 245 to go.\n",
      "bush.csv finished, 244 to go.\n",
      "rollerskates.csv finished, 243 to go.\n",
      "goatee.csv finished, 242 to go.\n",
      "cup.csv finished, 241 to go.\n",
      "anvil.csv finished, 240 to go.\n",
      "suitcase.csv finished, 239 to go.\n",
      "chair.csv finished, 238 to go.\n",
      "drill.csv finished, 237 to go.\n",
      "peanut.csv finished, 236 to go.\n",
      "squirrel.csv finished, 235 to go.\n",
      "matches.csv finished, 234 to go.\n",
      "sword.csv finished, 233 to go.\n",
      "cat.csv finished, 232 to go.\n",
      "toe.csv finished, 231 to go.\n",
      "snorkel.csv finished, 230 to go.\n",
      "pond.csv finished, 229 to go.\n",
      "calculator.csv finished, 228 to go.\n",
      "airplane.csv finished, 227 to go.\n",
      "squiggle.csv finished, 226 to go.\n",
      "blackberry.csv finished, 225 to go.\n",
      "ear.csv finished, 224 to go.\n",
      "frying pan.csv finished, 223 to go.\n",
      "chandelier.csv finished, 222 to go.\n",
      "lollipop.csv finished, 221 to go.\n",
      "binoculars.csv finished, 220 to go.\n",
      "garden.csv finished, 219 to go.\n",
      "basket.csv finished, 218 to go.\n",
      "penguin.csv finished, 217 to go.\n",
      "washing machine.csv finished, 216 to go.\n",
      "canoe.csv finished, 215 to go.\n",
      "screwdriver.csv finished, 214 to go.\n",
      "beach.csv finished, 213 to go.\n",
      "eyeglasses.csv finished, 212 to go.\n",
      "mouse.csv finished, 211 to go.\n",
      "apple.csv finished, 210 to go.\n",
      "van.csv finished, 209 to go.\n",
      "grapes.csv finished, 208 to go.\n",
      "grass.csv finished, 207 to go.\n",
      "watermelon.csv finished, 206 to go.\n",
      "floor lamp.csv finished, 205 to go.\n",
      "moon.csv finished, 204 to go.\n",
      "zigzag.csv finished, 203 to go.\n",
      "leg.csv finished, 202 to go.\n",
      "smiley face.csv finished, 201 to go.\n",
      "octagon.csv finished, 200 to go.\n",
      "dumbbell.csv finished, 199 to go.\n",
      "sweater.csv finished, 198 to go.\n",
      "stitches.csv finished, 197 to go.\n",
      "tractor.csv finished, 196 to go.\n",
      "foot.csv finished, 195 to go.\n",
      "helmet.csv finished, 194 to go.\n",
      "basketball.csv finished, 193 to go.\n",
      "crab.csv finished, 192 to go.\n",
      "clock.csv finished, 191 to go.\n",
      "diamond.csv finished, 190 to go.\n",
      "car.csv finished, 189 to go.\n",
      "axe.csv finished, 188 to go.\n",
      "traffic light.csv finished, 187 to go.\n",
      "sleeping bag.csv finished, 186 to go.\n",
      "baseball.csv finished, 185 to go.\n",
      "eye.csv finished, 184 to go.\n",
      "flower.csv finished, 183 to go.\n",
      "hot air balloon.csv finished, 182 to go.\n",
      "tree.csv finished, 181 to go.\n",
      "wine bottle.csv finished, 180 to go.\n",
      "hot tub.csv finished, 179 to go.\n",
      "peas.csv finished, 178 to go.\n",
      "door.csv finished, 177 to go.\n",
      "calendar.csv finished, 176 to go.\n",
      "wine glass.csv finished, 175 to go.\n",
      "stove.csv finished, 174 to go.\n",
      "hockey stick.csv finished, 173 to go.\n",
      "toothpaste.csv finished, 172 to go.\n",
      "moustache.csv finished, 171 to go.\n",
      "mountain.csv finished, 170 to go.\n",
      "tooth.csv finished, 169 to go.\n",
      "cannon.csv finished, 168 to go.\n",
      "firetruck.csv finished, 167 to go.\n",
      "shorts.csv finished, 166 to go.\n",
      "stereo.csv finished, 165 to go.\n",
      "cloud.csv finished, 164 to go.\n",
      "paintbrush.csv finished, 163 to go.\n",
      "pear.csv finished, 162 to go.\n",
      "dishwasher.csv finished, 161 to go.\n",
      "laptop.csv finished, 160 to go.\n",
      "frog.csv finished, 159 to go.\n",
      "vase.csv finished, 158 to go.\n",
      "diving board.csv finished, 157 to go.\n",
      "backpack.csv finished, 156 to go.\n",
      "lobster.csv finished, 155 to go.\n",
      "golf club.csv finished, 154 to go.\n",
      "garden hose.csv finished, 153 to go.\n",
      "hexagon.csv finished, 152 to go.\n",
      "bird.csv finished, 151 to go.\n",
      "finger.csv finished, 150 to go.\n",
      "animal migration.csv finished, 149 to go.\n",
      "steak.csv finished, 148 to go.\n",
      "mailbox.csv finished, 147 to go.\n",
      "shark.csv finished, 146 to go.\n",
      "television.csv finished, 145 to go.\n",
      "mermaid.csv finished, 144 to go.\n",
      "cow.csv finished, 143 to go.\n",
      "crayon.csv finished, 142 to go.\n",
      "palm tree.csv finished, 141 to go.\n",
      "windmill.csv finished, 140 to go.\n",
      "cookie.csv finished, 139 to go.\n",
      "kangaroo.csv finished, 138 to go.\n",
      "blueberry.csv finished, 137 to go.\n",
      "tiger.csv finished, 136 to go.\n",
      "tennis racquet.csv finished, 135 to go.\n",
      "dragon.csv finished, 134 to go.\n",
      "cell phone.csv finished, 133 to go.\n",
      "pineapple.csv finished, 132 to go.\n",
      "candle.csv finished, 131 to go.\n",
      "sheep.csv finished, 130 to go.\n",
      "cactus.csv finished, 129 to go.\n",
      "angel.csv finished, 128 to go.\n",
      "mosquito.csv finished, 127 to go.\n",
      "church.csv finished, 126 to go.\n",
      "couch.csv finished, 125 to go.\n",
      "The Great Wall of China.csv finished, 124 to go.\n",
      "tornado.csv finished, 123 to go.\n",
      "jacket.csv finished, 122 to go.\n",
      "nose.csv finished, 121 to go.\n",
      "octopus.csv finished, 120 to go.\n",
      "motorbike.csv finished, 119 to go.\n",
      "bracelet.csv finished, 118 to go.\n",
      "brain.csv finished, 117 to go.\n",
      "The Mona Lisa.csv finished, 116 to go.\n",
      "toothbrush.csv finished, 115 to go.\n",
      "carrot.csv finished, 114 to go.\n",
      "barn.csv finished, 113 to go.\n",
      "microphone.csv finished, 112 to go.\n",
      "zebra.csv finished, 111 to go.\n",
      "map.csv finished, 110 to go.\n",
      "camel.csv finished, 109 to go.\n",
      "wheel.csv finished, 108 to go.\n",
      "bridge.csv finished, 107 to go.\n",
      "lighthouse.csv finished, 106 to go.\n",
      "spreadsheet.csv finished, 105 to go.\n",
      "hockey puck.csv finished, 104 to go.\n",
      "wristwatch.csv finished, 103 to go.\n",
      "helicopter.csv finished, 102 to go.\n",
      "swan.csv finished, 101 to go.\n",
      "flamingo.csv finished, 100 to go.\n",
      "eraser.csv finished, 99 to go.\n",
      "bee.csv finished, 98 to go.\n",
      "flashlight.csv finished, 97 to go.\n",
      "megaphone.csv finished, 96 to go.\n",
      "ladder.csv finished, 95 to go.\n",
      "shoe.csv finished, 94 to go.\n",
      "asparagus.csv finished, 93 to go.\n",
      "t-shirt.csv finished, 92 to go.\n",
      "passport.csv finished, 91 to go.\n",
      "hand.csv finished, 90 to go.\n",
      "triangle.csv finished, 89 to go.\n",
      "lightning.csv finished, 88 to go.\n",
      "mug.csv finished, 87 to go.\n",
      "submarine.csv finished, 86 to go.\n",
      "violin.csv finished, 85 to go.\n",
      "owl.csv finished, 84 to go.\n",
      "scissors.csv finished, 83 to go.\n",
      "baseball bat.csv finished, 82 to go.\n",
      "string bean.csv finished, 81 to go.\n",
      "lantern.csv finished, 80 to go.\n",
      "house.csv finished, 79 to go.\n",
      "elbow.csv finished, 78 to go.\n",
      "power outlet.csv finished, 77 to go.\n",
      "stop sign.csv finished, 76 to go.\n",
      "bed.csv finished, 75 to go.\n",
      "school bus.csv finished, 74 to go.\n",
      "hamburger.csv finished, 73 to go.\n",
      "lipstick.csv finished, 72 to go.\n",
      "light bulb.csv finished, 71 to go.\n",
      "flip flops.csv finished, 70 to go.\n",
      "alarm clock.csv finished, 69 to go.\n",
      "ant.csv finished, 68 to go.\n",
      "face.csv finished, 67 to go.\n",
      "microwave.csv finished, 66 to go.\n",
      "hourglass.csv finished, 65 to go.\n",
      "panda.csv finished, 64 to go.\n",
      "pool.csv finished, 63 to go.\n",
      "circle.csv finished, 62 to go.\n",
      "onion.csv finished, 61 to go.\n",
      "raccoon.csv finished, 60 to go.\n",
      "bowtie.csv finished, 59 to go.\n",
      "umbrella.csv finished, 58 to go.\n",
      "butterfly.csv finished, 57 to go.\n",
      "fireplace.csv finished, 56 to go.\n",
      "skull.csv finished, 55 to go.\n",
      "train.csv finished, 54 to go.\n",
      "mouth.csv finished, 53 to go.\n",
      "hat.csv finished, 52 to go.\n",
      "drums.csv finished, 51 to go.\n",
      "book.csv finished, 50 to go.\n",
      "radio.csv finished, 49 to go.\n",
      "roller coaster.csv finished, 48 to go.\n",
      "snowflake.csv finished, 47 to go.\n",
      "piano.csv finished, 46 to go.\n",
      "rhinoceros.csv finished, 45 to go.\n",
      "cake.csv finished, 44 to go.\n",
      "toaster.csv finished, 43 to go.\n",
      "paint can.csv finished, 42 to go.\n",
      "knee.csv finished, 41 to go.\n",
      "spider.csv finished, 40 to go.\n",
      "tent.csv finished, 39 to go.\n",
      "rabbit.csv finished, 38 to go.\n",
      "clarinet.csv finished, 37 to go.\n",
      "whale.csv finished, 36 to go.\n",
      "boomerang.csv finished, 35 to go.\n",
      "hospital.csv finished, 34 to go.\n",
      "ceiling fan.csv finished, 33 to go.\n",
      "saw.csv finished, 32 to go.\n",
      "pillow.csv finished, 31 to go.\n",
      "fence.csv finished, 30 to go.\n",
      "dog.csv finished, 29 to go.\n",
      "duck.csv finished, 28 to go.\n",
      "parrot.csv finished, 27 to go.\n",
      "swing set.csv finished, 26 to go.\n",
      "spoon.csv finished, 25 to go.\n",
      "fan.csv finished, 24 to go.\n",
      "cruise ship.csv finished, 23 to go.\n",
      "picture frame.csv finished, 22 to go.\n",
      "mushroom.csv finished, 21 to go.\n",
      "headphones.csv finished, 20 to go.\n",
      "horse.csv finished, 19 to go.\n",
      "flying saucer.csv finished, 18 to go.\n",
      "lion.csv finished, 17 to go.\n",
      "postcard.csv finished, 16 to go.\n",
      "bench.csv finished, 15 to go.\n",
      "keyboard.csv finished, 14 to go.\n",
      "parachute.csv finished, 13 to go.\n",
      "streetlight.csv finished, 12 to go.\n",
      "arm.csv finished, 11 to go.\n",
      "police car.csv finished, 10 to go.\n",
      "sailboat.csv finished, 9 to go.\n",
      "cooler.csv finished, 8 to go.\n",
      "bathtub.csv finished, 7 to go.\n",
      "campfire.csv finished, 6 to go.\n",
      "hurricane.csv finished, 5 to go.\n",
      "soccer ball.csv finished, 4 to go.\n",
      "potato.csv finished, 3 to go.\n",
      "dolphin.csv finished, 2 to go.\n",
      "key.csv finished, 1 to go.\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "# Filter the dataset so that the cleaned dataset only contains sketching \n",
    "# which are successfully recognized and from US or China.\n",
    "df = pd.read_csv(path + files[0])\n",
    "df = pd.read_csv(path + files[0])\n",
    "df['recognized'] = df['recognized'].astype(bool)\n",
    "df = df[(df['countrycode'] == 'US') & (df['recognized'] == True)]\n",
    "df = df[:1000]\n",
    "\n",
    "i = 1\n",
    "for file_i in files[1:]:\n",
    "  df_i = pd.read_csv(path + file_i)\n",
    "  df_i['recognized'] = df_i['recognized'].astype(bool)\n",
    "  df_i = df_i[(df_i['countrycode'] == 'US') & (df_i['recognized'] == True)]\n",
    "  df_i = df_i[:1000]\n",
    "\n",
    "  df = df.append(df_i)\n",
    "  print(file_i + ' finished, ' + str(len(files) - i) + ' to go.')\n",
    "  i += 1  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "3ww_yO48IQMX"
   },
   "outputs": [],
   "source": [
    "# Save cleaned dataset\n",
    "df.to_csv('us_all_1000.csv')\n",
    "!cp './us_all_1000.csv' '/content/drive/My Drive/590-cleaned-data/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "t_0E-cudIN-6"
   },
   "source": [
    "## 3. Convert raw strokes into images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "I8eNT0o_-Iux"
   },
   "outputs": [],
   "source": [
    "# Define a function to convert raw strokes into images with 256 by 256 pixels\n",
    "def draw_cv2(raw_strokes, size=256, lw=3):   \n",
    "    step = len(raw_strokes)\n",
    "    max_x = max( [max(x)  for x, y in raw_strokes ] )\n",
    "    max_y = max( [max(y)  for x, y in raw_strokes ] )   \n",
    "    max_x, max_y =  max_x + 1, max_y + 1\n",
    "    img = np.zeros((max_y, max_x), np.uint8)   \n",
    "    for i in range(step):\n",
    "        stroke = raw_strokes[i]        \n",
    "        x, y=stroke[0], stroke[1]\n",
    "        for i in range(len(x) - 1):\n",
    "            _ = cv2.line(img, (x[i], y[i]),(x[i + 1], y[i + 1]), 255, lw)   \n",
    "    top = max ((size- max_y) // 2 , 0 )\n",
    "    bottom = max ( size- top - max_y , 0)\n",
    "    left = max ((size - max_x) // 2, 0 )\n",
    "    right = max (size - max_x - left, 0 )\n",
    "    img =  cv2.copyMakeBorder(img,top,bottom,left,right,cv2.BORDER_CONSTANT)\n",
    "    return img"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "RhHJvrla-Ixw",
    "outputId": "54027e91-8fe9-4427-cf54-0d47f18e3f24"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(341000, 1, 28, 28, 1)\n",
      "(341000, 1, 340)\n"
     ]
    }
   ],
   "source": [
    "# Prepare training and validate dataset for the LSTM+CNN model\n",
    "import pandas as pd\n",
    "import ast\n",
    "import cv2\n",
    "from keras.utils import np_utils\n",
    "import numpy as np\n",
    "df= pd.read_csv('/content/drive/My Drive/590-cleaned-data/us_all_1000.csv')\n",
    "X_train = []\n",
    "y_train = pd.Categorical(df['word'], categories=df['word'].unique()).codes\n",
    "df['word'].nunique()\n",
    "Y_train = np_utils.to_categorical(y_train, 340)\n",
    "df['drawing'] = df['drawing'].map(ast.literal_eval)\n",
    "\n",
    "# Due to computation and memory limitation, we need to shrink the size of the original 256 by 256 images\n",
    "for i in range(len(df)):\n",
    "    image = draw_cv2(df['drawing'][i], size=256, lw=3)\n",
    "    # Shrink the size into 28 by 28\n",
    "    image = cv2.resize(image, (28,28), interpolation = cv2.INTER_CUBIC)\n",
    "    X_train.append(image)\n",
    "    \n",
    "# Convert the dataset into 5 dimensions for the LSTM+CNN model\n",
    "X_train = np.array(X_train).reshape(len(X_train), 1, 28, 28,1).astype('float32') / 255\n",
    "Y_train = Y_train.reshape(len(Y_train), 1,340)\n",
    "print(X_train.shape)\n",
    "print(Y_train.shape)\n",
    "\n",
    "# Split into training and validate datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "train_X, valid_X, train_Y, valid_Y = train_test_split(X_train,Y_train,test_size=0.2, random_state=13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "5qR-1uvx6B8d",
    "outputId": "094edffd-61ec-43d2-e43c-b6f942854eb4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(341000, 28, 28, 1)\n",
      "(341000, 340)\n"
     ]
    }
   ],
   "source": [
    "# Prepare training and validate dataset for the LSTM+CNN model\n",
    "import pandas as pd\n",
    "import ast\n",
    "import cv2\n",
    "from keras.utils import np_utils\n",
    "import numpy as np\n",
    "df= pd.read_csv('/content/drive/My Drive/590-cleaned-data/us_all_1000.csv')\n",
    "X_train = []\n",
    "y_train = pd.Categorical(df['word'], categories=df['word'].unique()).codes\n",
    "df['word'].nunique()\n",
    "Y_train = np_utils.to_categorical(y_train, 340)\n",
    "df['drawing'] = df['drawing'].map(ast.literal_eval)\n",
    "\n",
    "# Due to computation and memory limitation, we need to shrink the size of the original 256 by 256 images\n",
    "for i in range(len(df)):\n",
    "    image = draw_cv2(df['drawing'][i], size=256, lw=3)\n",
    "    # Shrink the size into 28 by 28\n",
    "    image = cv2.resize(image, (28,28), interpolation = cv2.INTER_CUBIC)\n",
    "    X_train.append(image)\n",
    "    \n",
    "# Convert the dataset into 4 dimensions for the basic CNN model    \n",
    "X_train = np.array(X_train).reshape(len(X_train),28,28,1).astype('float32') / 255\n",
    "print(X_train.shape)\n",
    "print(Y_train.shape)\n",
    "\n",
    "# Split into training and validate datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "train_X, valid_X, train_Y, valid_Y = train_test_split(X_train,Y_train,test_size=0.2, random_state=13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "1OnTNylf2ebS"
   },
   "outputs": [],
   "source": [
    "from keras.layers import Input, Conv2D, MaxPooling2D, UpSampling2D\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.layers import SimpleRNN, LSTM, Embedding, GRU\n",
    "from keras.layers import TimeDistributed, Lambda\n",
    "from keras.layers import Conv1D, MaxPooling1D\n",
    "from keras.models import Model, Sequential\n",
    "from keras.preprocessing import sequence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "N2HoqXh_5DMR"
   },
   "source": [
    "## 4. The basic CNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 193
    },
    "colab_type": "code",
    "id": "tPPiWPPc5CLT",
    "outputId": "89c4f42f-97a1-4941-c36f-c9b93701a31c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1259: calling reduce_prod (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:2880: calling reduce_sum (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1344: calling reduce_mean (from tensorflow.python.ops.math_ops) with keep_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "keep_dims is deprecated, use keepdims instead\n"
     ]
    }
   ],
   "source": [
    "# Basic CNN model: 2 Conv2D layers, 1 maxpooling layer, 2 drop out layers and 2 full connection layers\n",
    "model = Sequential()\n",
    "model.add(Conv2D(filters = 32,kernel_size=(3, 3), \n",
    "                 activation='relu', strides=(1, 1), \n",
    "                 padding='valid',\n",
    "                 input_shape=(28,28,1)))\n",
    "model.add(Conv2D(filters = 32,kernel_size=(3, 3), \n",
    "                 activation='relu', strides=(1, 1), \n",
    "                 padding='valid'))\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(Dropout(0.7))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(256, activation='relu'))\n",
    "model.add(Dropout(0.7))\n",
    "model.add(Dense(340, activation='softmax'))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 3462
    },
    "colab_type": "code",
    "id": "OFFlqq017uVc",
    "outputId": "38de1970-f4db-4048-b370-7c297a6b872a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 272800 samples, validate on 68200 samples\n",
      "Epoch 1/80\n",
      "272800/272800 [==============================] - 22s 79us/step - loss: 5.0143 - acc: 0.0545 - val_loss: 3.8024 - val_acc: 0.2352\n",
      "Epoch 2/80\n",
      "272800/272800 [==============================] - 18s 68us/step - loss: 3.9794 - acc: 0.1601 - val_loss: 3.2089 - val_acc: 0.3136\n",
      "Epoch 3/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 3.6528 - acc: 0.2070 - val_loss: 2.9792 - val_acc: 0.3479\n",
      "Epoch 4/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 3.5004 - acc: 0.2293 - val_loss: 2.8427 - val_acc: 0.3721\n",
      "Epoch 5/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 3.4039 - acc: 0.2452 - val_loss: 2.7759 - val_acc: 0.3834\n",
      "Epoch 6/80\n",
      "272800/272800 [==============================] - 19s 68us/step - loss: 3.3443 - acc: 0.2538 - val_loss: 2.7277 - val_acc: 0.3935\n",
      "Epoch 7/80\n",
      "272800/272800 [==============================] - 19s 68us/step - loss: 3.2979 - acc: 0.2616 - val_loss: 2.6614 - val_acc: 0.4003\n",
      "Epoch 8/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 3.2557 - acc: 0.2678 - val_loss: 2.6173 - val_acc: 0.4109\n",
      "Epoch 9/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 3.2254 - acc: 0.2742 - val_loss: 2.5933 - val_acc: 0.4164\n",
      "Epoch 10/80\n",
      "272800/272800 [==============================] - 19s 71us/step - loss: 3.2022 - acc: 0.2773 - val_loss: 2.5634 - val_acc: 0.4208\n",
      "Epoch 11/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 3.1805 - acc: 0.2814 - val_loss: 2.5734 - val_acc: 0.4190\n",
      "Epoch 12/80\n",
      "272800/272800 [==============================] - 19s 71us/step - loss: 3.1620 - acc: 0.2849 - val_loss: 2.5407 - val_acc: 0.4250\n",
      "Epoch 13/80\n",
      "272800/272800 [==============================] - 20s 72us/step - loss: 3.1471 - acc: 0.2868 - val_loss: 2.5075 - val_acc: 0.4307\n",
      "Epoch 14/80\n",
      "272800/272800 [==============================] - 19s 71us/step - loss: 3.1287 - acc: 0.2887 - val_loss: 2.4777 - val_acc: 0.4323\n",
      "Epoch 15/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 3.1166 - acc: 0.2917 - val_loss: 2.4639 - val_acc: 0.4344\n",
      "Epoch 16/80\n",
      "272800/272800 [==============================] - 18s 68us/step - loss: 3.1050 - acc: 0.2946 - val_loss: 2.4597 - val_acc: 0.4379\n",
      "Epoch 17/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 3.0919 - acc: 0.2964 - val_loss: 2.4482 - val_acc: 0.4385\n",
      "Epoch 18/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 3.0834 - acc: 0.2967 - val_loss: 2.4525 - val_acc: 0.4391\n",
      "Epoch 19/80\n",
      "272800/272800 [==============================] - 19s 71us/step - loss: 3.0700 - acc: 0.2992 - val_loss: 2.4227 - val_acc: 0.4433\n",
      "Epoch 20/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 3.0605 - acc: 0.3008 - val_loss: 2.4360 - val_acc: 0.4429\n",
      "Epoch 21/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 3.0587 - acc: 0.3017 - val_loss: 2.4069 - val_acc: 0.4471\n",
      "Epoch 22/80\n",
      "272800/272800 [==============================] - 19s 71us/step - loss: 3.0480 - acc: 0.3032 - val_loss: 2.3998 - val_acc: 0.4472\n",
      "Epoch 23/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 3.0424 - acc: 0.3048 - val_loss: 2.4028 - val_acc: 0.4492\n",
      "Epoch 24/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 3.0292 - acc: 0.3065 - val_loss: 2.3768 - val_acc: 0.4524\n",
      "Epoch 25/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 3.0286 - acc: 0.3057 - val_loss: 2.3909 - val_acc: 0.4509\n",
      "Epoch 26/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 3.0172 - acc: 0.3081 - val_loss: 2.3773 - val_acc: 0.4504\n",
      "Epoch 27/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 3.0143 - acc: 0.3096 - val_loss: 2.3750 - val_acc: 0.4526\n",
      "Epoch 28/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 3.0107 - acc: 0.3096 - val_loss: 2.3565 - val_acc: 0.4535\n",
      "Epoch 29/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 3.0035 - acc: 0.3109 - val_loss: 2.3714 - val_acc: 0.4548\n",
      "Epoch 30/80\n",
      "272800/272800 [==============================] - 19s 71us/step - loss: 2.9989 - acc: 0.3113 - val_loss: 2.3597 - val_acc: 0.4554\n",
      "Epoch 31/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 2.9927 - acc: 0.3112 - val_loss: 2.3492 - val_acc: 0.4587\n",
      "Epoch 32/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 2.9890 - acc: 0.3131 - val_loss: 2.3589 - val_acc: 0.4563\n",
      "Epoch 33/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 2.9850 - acc: 0.3137 - val_loss: 2.3508 - val_acc: 0.4596\n",
      "Epoch 34/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 2.9793 - acc: 0.3138 - val_loss: 2.3392 - val_acc: 0.4592\n",
      "Epoch 35/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 2.9722 - acc: 0.3159 - val_loss: 2.3279 - val_acc: 0.4598\n",
      "Epoch 36/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 2.9713 - acc: 0.3156 - val_loss: 2.3219 - val_acc: 0.4607\n",
      "Epoch 37/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 2.9665 - acc: 0.3158 - val_loss: 2.3166 - val_acc: 0.4617\n",
      "Epoch 38/80\n",
      "272800/272800 [==============================] - 19s 71us/step - loss: 2.9637 - acc: 0.3167 - val_loss: 2.3121 - val_acc: 0.4620\n",
      "Epoch 39/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 2.9656 - acc: 0.3174 - val_loss: 2.3314 - val_acc: 0.4621\n",
      "Epoch 40/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 2.9580 - acc: 0.3175 - val_loss: 2.3167 - val_acc: 0.4626\n",
      "Epoch 41/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 2.9509 - acc: 0.3191 - val_loss: 2.2972 - val_acc: 0.4646\n",
      "Epoch 42/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 2.9500 - acc: 0.3205 - val_loss: 2.3097 - val_acc: 0.4654\n",
      "Epoch 43/80\n",
      "272800/272800 [==============================] - 19s 71us/step - loss: 2.9520 - acc: 0.3189 - val_loss: 2.2993 - val_acc: 0.4640\n",
      "Epoch 44/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 2.9411 - acc: 0.3206 - val_loss: 2.3024 - val_acc: 0.4637\n",
      "Epoch 45/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 2.9437 - acc: 0.3206 - val_loss: 2.2791 - val_acc: 0.4687\n",
      "Epoch 46/80\n",
      "272800/272800 [==============================] - 19s 71us/step - loss: 2.9357 - acc: 0.3214 - val_loss: 2.2891 - val_acc: 0.4680\n",
      "Epoch 47/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 2.9367 - acc: 0.3217 - val_loss: 2.3087 - val_acc: 0.4640\n",
      "Epoch 48/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 2.9336 - acc: 0.3220 - val_loss: 2.2905 - val_acc: 0.4689\n",
      "Epoch 49/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 2.9320 - acc: 0.3220 - val_loss: 2.2675 - val_acc: 0.4692\n",
      "Epoch 50/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 2.9283 - acc: 0.3231 - val_loss: 2.2894 - val_acc: 0.4680\n",
      "Epoch 51/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 2.9269 - acc: 0.3228 - val_loss: 2.2759 - val_acc: 0.4693\n",
      "Epoch 52/80\n",
      "272800/272800 [==============================] - 19s 71us/step - loss: 2.9231 - acc: 0.3236 - val_loss: 2.2627 - val_acc: 0.4710\n",
      "Epoch 53/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 2.9146 - acc: 0.3253 - val_loss: 2.2931 - val_acc: 0.4707\n",
      "Epoch 54/80\n",
      "272800/272800 [==============================] - 19s 69us/step - loss: 2.9220 - acc: 0.3238 - val_loss: 2.2906 - val_acc: 0.4660\n",
      "Epoch 55/80\n",
      "272800/272800 [==============================] - 19s 70us/step - loss: 2.9139 - acc: 0.3252 - val_loss: 2.2657 - val_acc: 0.4707\n",
      "Epoch 56/80\n",
      "272800/272800 [==============================] - 19s 71us/step - loss: 2.9165 - acc: 0.3244 - val_loss: 2.2732 - val_acc: 0.4707\n",
      "Epoch 57/80\n",
      "272800/272800 [==============================] - 20s 71us/step - loss: 2.9132 - acc: 0.3252 - val_loss: 2.2618 - val_acc: 0.4728\n",
      "Epoch 58/80\n",
      "150528/272800 [===============>..............] - ETA: 7s - loss: 2.9040 - acc: 0.3274"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-241c5688c123>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m          \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m80\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m          \u001b[0mbatch_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1024\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m         validation_data=(valid_X, valid_Y))\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/models.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m    958\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    959\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 960\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m    961\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    962\u001b[0m     def evaluate(self, x, y, batch_size=32, verbose=1,\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1646\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1647\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1648\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1649\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1650\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1211\u001b[0m                     \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1212\u001b[0m                     \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1213\u001b[0;31m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1214\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1215\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2350\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2351\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2352\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2353\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2354\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    927\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    928\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 929\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    930\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    931\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1150\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1151\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1152\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1153\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1154\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1327\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1328\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1329\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1330\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1332\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1333\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1334\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1335\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1336\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1317\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1319\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1320\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1321\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1405\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1406\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1407\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1408\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1409\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Training the CNN model\n",
    "model.fit(train_X, train_Y,\n",
    "         epochs = 80,\n",
    "         batch_size = 1024,\n",
    "        validation_data=(valid_X, valid_Y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "h7h9ux6V5GY2"
   },
   "source": [
    "## 5. The LSTM+CNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "vSxkQKqRiwDQ"
   },
   "outputs": [],
   "source": [
    "# The LSTM+CNN model: 2 Conv2D layers, 1 maxpooling layer, 1 dropout layer, 1 LSTM layer and 2 full connection layers\n",
    "model = Sequential()\n",
    "model.add(TimeDistributed(Conv2D(64, (3, 3), activation='relu'),\n",
    "                          input_shape=(None, 28, 28, 1)))\n",
    "model.add(TimeDistributed(MaxPooling2D((2, 2), strides=(1, 1))))\n",
    "model.add(TimeDistributed(Conv2D(32, (3,3), activation='relu')))\n",
    "model.add(TimeDistributed(Dropout(0.8)))\n",
    "# Convert the 2D ouput into one sequential vector as input of the LSTM layer\n",
    "model.add(TimeDistributed(Flatten()))\n",
    "# define LSTM model\n",
    "model.add(LSTM(128, return_sequences=True))\n",
    "model.add(Dense(256, activation = 'relu'))\n",
    "model.add(Dense(340, activation = 'softmax'))\n",
    "model.compile(loss = 'categorical_crossentropy',\n",
    "             optimizer = 'adam',\n",
    "             metrics = ['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ilhs0jZ55Ey7"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "collapsed": true,
    "id": "h1KyIUQgNXPy"
   },
   "outputs": [],
   "source": [
    "model.compile(loss = 'categorical_crossentropy',\n",
    "             optimizer = 'adam',\n",
    "             metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1248
    },
    "colab_type": "code",
    "id": "WQq6fB4ZNYv8",
    "outputId": "4621d5b8-ab0b-4cad-926c-8e1e11a54259"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 272800 samples, validate on 68200 samples\n",
      "Epoch 1/60\n",
      "272800/272800 [==============================] - 50s 182us/step - loss: 4.1019 - acc: 0.1621 - val_loss: 3.1080 - val_acc: 0.2994\n",
      "Epoch 2/60\n",
      "272800/272800 [==============================] - 46s 167us/step - loss: 2.9691 - acc: 0.3198 - val_loss: 2.6984 - val_acc: 0.3670\n",
      "Epoch 3/60\n",
      "272800/272800 [==============================] - 46s 167us/step - loss: 2.6517 - acc: 0.3733 - val_loss: 2.4429 - val_acc: 0.4159\n",
      "Epoch 4/60\n",
      "272800/272800 [==============================] - 45s 166us/step - loss: 2.4692 - acc: 0.4061 - val_loss: 2.3285 - val_acc: 0.4336\n",
      "Epoch 5/60\n",
      "272800/272800 [==============================] - 46s 167us/step - loss: 2.3439 - acc: 0.4291 - val_loss: 2.2508 - val_acc: 0.4478\n",
      "Epoch 6/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 2.2475 - acc: 0.4470 - val_loss: 2.1913 - val_acc: 0.4614\n",
      "Epoch 7/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 2.1640 - acc: 0.4634 - val_loss: 2.1257 - val_acc: 0.4732\n",
      "Epoch 8/60\n",
      "272800/272800 [==============================] - 45s 166us/step - loss: 2.0973 - acc: 0.4762 - val_loss: 2.0813 - val_acc: 0.4822\n",
      "Epoch 9/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 2.0390 - acc: 0.4873 - val_loss: 2.0566 - val_acc: 0.4896\n",
      "Epoch 10/60\n",
      "272800/272800 [==============================] - 46s 169us/step - loss: 1.9873 - acc: 0.4974 - val_loss: 2.0225 - val_acc: 0.4955\n",
      "Epoch 11/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 1.9400 - acc: 0.5080 - val_loss: 1.9876 - val_acc: 0.5030\n",
      "Epoch 12/60\n",
      "272800/272800 [==============================] - 45s 166us/step - loss: 1.8999 - acc: 0.5151 - val_loss: 1.9781 - val_acc: 0.5051\n",
      "Epoch 13/60\n",
      "272800/272800 [==============================] - 46s 167us/step - loss: 1.8625 - acc: 0.5232 - val_loss: 1.9507 - val_acc: 0.5118\n",
      "Epoch 14/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 1.8299 - acc: 0.5288 - val_loss: 1.9526 - val_acc: 0.5101\n",
      "Epoch 15/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 1.7943 - acc: 0.5367 - val_loss: 1.9262 - val_acc: 0.5148\n",
      "Epoch 16/60\n",
      "272800/272800 [==============================] - 45s 166us/step - loss: 1.7668 - acc: 0.5423 - val_loss: 1.9157 - val_acc: 0.5191\n",
      "Epoch 17/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 1.7398 - acc: 0.5465 - val_loss: 1.9153 - val_acc: 0.5185\n",
      "Epoch 18/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 1.7138 - acc: 0.5521 - val_loss: 1.9127 - val_acc: 0.5227\n",
      "Epoch 19/60\n",
      "272800/272800 [==============================] - 46s 167us/step - loss: 1.6878 - acc: 0.5570 - val_loss: 1.8997 - val_acc: 0.5257\n",
      "Epoch 20/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 1.6675 - acc: 0.5615 - val_loss: 1.9075 - val_acc: 0.5232\n",
      "Epoch 21/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 1.6462 - acc: 0.5668 - val_loss: 1.8867 - val_acc: 0.5281\n",
      "Epoch 22/60\n",
      "272800/272800 [==============================] - 46s 167us/step - loss: 1.6274 - acc: 0.5703 - val_loss: 1.8904 - val_acc: 0.5271\n",
      "Epoch 23/60\n",
      "272800/272800 [==============================] - 46s 167us/step - loss: 1.6053 - acc: 0.5741 - val_loss: 1.8763 - val_acc: 0.5315\n",
      "Epoch 24/60\n",
      "272800/272800 [==============================] - 46s 167us/step - loss: 1.5868 - acc: 0.5780 - val_loss: 1.8954 - val_acc: 0.5270\n",
      "Epoch 25/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 1.5733 - acc: 0.5816 - val_loss: 1.8621 - val_acc: 0.5340\n",
      "Epoch 26/60\n",
      "272800/272800 [==============================] - 45s 166us/step - loss: 1.5587 - acc: 0.5848 - val_loss: 1.8673 - val_acc: 0.5332\n",
      "Epoch 27/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 1.5352 - acc: 0.5899 - val_loss: 1.8885 - val_acc: 0.5314\n",
      "Epoch 28/60\n",
      "272800/272800 [==============================] - 46s 169us/step - loss: 1.5208 - acc: 0.5915 - val_loss: 1.8652 - val_acc: 0.5336\n",
      "Epoch 29/60\n",
      "272800/272800 [==============================] - 45s 166us/step - loss: 1.5085 - acc: 0.5944 - val_loss: 1.8725 - val_acc: 0.5359\n",
      "Epoch 30/60\n",
      "272800/272800 [==============================] - 46s 167us/step - loss: 1.4931 - acc: 0.5981 - val_loss: 1.8730 - val_acc: 0.5349\n",
      "Epoch 31/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 1.4800 - acc: 0.6011 - val_loss: 1.8798 - val_acc: 0.5353\n",
      "Epoch 32/60\n",
      "272800/272800 [==============================] - 46s 168us/step - loss: 1.4673 - acc: 0.6031 - val_loss: 1.8953 - val_acc: 0.5325\n",
      "Epoch 33/60\n",
      "272800/272800 [==============================] - 46s 167us/step - loss: 1.4517 - acc: 0.6071 - val_loss: 1.8924 - val_acc: 0.5338\n",
      "Epoch 34/60\n",
      "272800/272800 [==============================] - 46s 170us/step - loss: 1.4395 - acc: 0.6098 - val_loss: 1.8815 - val_acc: 0.5361\n",
      "Epoch 35/60\n",
      "239616/272800 [=========================>....] - ETA: 5s - loss: 1.4275 - acc: 0.6115"
     ]
    }
   ],
   "source": [
    "# Training the LSTM+CNN model\n",
    "model.fit(train_X, train_Y,\n",
    "         epochs = 60,\n",
    "         batch_size = 1024,\n",
    "        validation_data=(valid_X, valid_Y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6AfE2qWAGOqF"
   },
   "source": [
    "## 6. Use the test data to test the performance of  the basic CNN model and LSTM+CNN model. In this section, we does cross test here which means that when training on US dataset, then we will test the model on Chinese dataset , and vice versa."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 69
    },
    "colab_type": "code",
    "id": "9_oczghJPmBr",
    "outputId": "66299328-6fa6-4412-a0d7-d08224c4dd72"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5138, 1, 28, 28, 1)\n",
      "(5138,)\n",
      "(5138, 1, 340)\n"
     ]
    }
   ],
   "source": [
    "# Read in the test dataset for the LSTM+CNN model\n",
    "# When training on US data, the Chinese sketching data is test data\n",
    "df_cn = pd.read_csv('/content/drive/My Drive/590-cleaned-data/df_cn.csv')\n",
    "X_test = []\n",
    "y_test = pd.Categorical(df_cn['word'], categories=df['word'].unique()).codes\n",
    "Y_test = np_utils.to_categorical(y_test, 340)\n",
    "df_cn['drawing'] = df_cn['drawing'].map(ast.literal_eval)\n",
    "for i in range(len(df_cn)):\n",
    "    image = draw_cv2(df_cn['drawing'][i], size=256, lw=3)\n",
    "    image = cv2.resize(image, (28,28), interpolation = cv2.INTER_CUBIC)\n",
    "    X_test.append(image)\n",
    "X_test = np.array(X_test).reshape(len(X_test), 1, 28,28,1).astype('float32') / 255\n",
    "Y_test = Y_test.reshape(len(Y_test), 1,340)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)\n",
    "print(Y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 69
    },
    "colab_type": "code",
    "id": "-ufs7rYK-4Yo",
    "outputId": "6557fd0b-f0a4-449f-99c3-01e1c8f21aac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5138, 28, 28, 1)\n",
      "(5138,)\n",
      "(5138, 340)\n"
     ]
    }
   ],
   "source": [
    "# Read in the test dataset for the CNN model\n",
    "# When training on US data, the Chinese sketching data is test data\n",
    "import pandas as pd\n",
    "import ast\n",
    "import cv2\n",
    "from keras.utils import np_utils\n",
    "import numpy as np\n",
    "df_cn = pd.read_csv('/content/drive/My Drive/590-cleaned-data/df_cn.csv')\n",
    "X_test = []\n",
    "y_test = pd.Categorical(df_cn['word'], categories=df['word'].unique()).codes\n",
    "Y_test = np_utils.to_categorical(y_test, 340)\n",
    "df_cn['drawing'] = df_cn['drawing'].map(ast.literal_eval)\n",
    "for i in range(len(df_cn)):\n",
    "    image = draw_cv2(df_cn['drawing'][i], size=256, lw=3)\n",
    "    image = cv2.resize(image, (28,28), interpolation = cv2.INTER_CUBIC)\n",
    "    X_test.append(image)\n",
    "X_test = np.array(X_test).reshape(len(X_test), 28, 28,1).astype('float32') / 255\n",
    "#Y_test = Y_test.reshape(len(Y_test), 1,340)\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)\n",
    "print(Y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "BUMecRriPse8",
    "outputId": "17b98cd8-f5aa-47e6-e56c-799702aae976"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5138/5138 [==============================] - 1s 190us/step\n",
      "[2.616116162170553, 0.42604126119112495]\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(X_test, Y_test, verbose = 1)\n",
    "print(score)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "590_process_data.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}