Word Language Modelling LSTM .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "Using gpu device 0: GeForce GT 755M (CNMeM is enabled with initial size: 80.0% of memory, cuDNN not available)\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import sys\n",
    "import string\n",
    "import numpy as np\n",
    "from keras.utils import np_utils\n",
    "from keras.models import Sequential\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.layers import Dense, Dropout, LSTM\n",
    "from keras.layers.embeddings import Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rawtext = open('wonderland.txt','r').read().split('\\n')\n",
    "rawtext = ' '.join(rawtext)\n",
    "rawtext = [word.strip(string.punctuation) for word in rawtext.split()]\n",
    "rawtext = ' '.join(rawtext)\n",
    "rawtext = rawtext.replace('-', ' ')\n",
    "rawtext = ' '.join(rawtext.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Vocab: 3063\n"
     ]
    }
   ],
   "source": [
    "all_words = rawtext.split()\n",
    "unique_words = sorted(list(set(all_words)))\n",
    "n_vocab = len(unique_words)\n",
    "print \"Total Vocab:\", n_vocab\n",
    "word_to_int = dict((w, i) for i, w in enumerate(unique_words))\n",
    "int_to_word = dict((i, w) for i, w in enumerate(unique_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26694\n"
     ]
    }
   ],
   "source": [
    "raw_text = rawtext.split()\n",
    "n_words = len(raw_text)\n",
    "print n_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total patterns: 26594\n"
     ]
    }
   ],
   "source": [
    "seq_length = 100\n",
    "dataX = []\n",
    "dataY = []\n",
    "for i in xrange(0, n_words - seq_length):\n",
    "    seq_in  = raw_text[i: i+seq_length]\n",
    "    seq_out = raw_text[i+seq_length]\n",
    "    dataX.append([word_to_int[word] for word in seq_in])\n",
    "    dataY.append(word_to_int[seq_out])\n",
    "n_patterns = len(dataX)\n",
    "print 'Total patterns:', n_patterns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Reshape dataX to size of [samples, time steps, features] and scale it to 0-1\n",
    "# Represent dataY as one hot encoding\n",
    "X_train = np.reshape(dataX, (n_patterns, seq_length, 1))/float(n_vocab)\n",
    "Y_train = np_utils.to_categorical(dataY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "lstm_1 (LSTM)                    (None, 256)           264192      lstm_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 256)           0           lstm_1[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 3063L)         787191      dropout_1[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 1051383\n",
      "____________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(LSTM(256, input_shape=(X_train.shape[1], X_train.shape[2])))\n",
    "model.add(Dropout(0.2))\n",
    "model.add(Dense(Y_train.shape[1], activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
    "print model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define the checkpoint\n",
    "filepath=\"data/weights/word-weights-improvement-{epoch:02d}-{loss:.4f}.hdf5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')\n",
    "callbacks_list = [checkpoint]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.3193Epoch 00000: loss did not improve\n",
      "26594/26594 [==============================] - 143s - loss: 6.3193   \n",
      "Epoch 2/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.2405Epoch 00001: loss improved from 6.24777 to 6.24087, saving model to weights-improvement-01-6.2409.hdf5\n",
      "26594/26594 [==============================] - 143s - loss: 6.2409   \n",
      "Epoch 3/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.2085Epoch 00002: loss improved from 6.24087 to 6.20792, saving model to weights-improvement-02-6.2079.hdf5\n",
      "26594/26594 [==============================] - 143s - loss: 6.2079   \n",
      "Epoch 4/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.1729Epoch 00003: loss improved from 6.20792 to 6.17167, saving model to weights-improvement-03-6.1717.hdf5\n",
      "26594/26594 [==============================] - 142s - loss: 6.1717   \n",
      "Epoch 5/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.1348Epoch 00004: loss improved from 6.17167 to 6.13402, saving model to weights-improvement-04-6.1340.hdf5\n",
      "26594/26594 [==============================] - 141s - loss: 6.1340   \n",
      "Epoch 6/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.0864Epoch 00005: loss improved from 6.13402 to 6.08678, saving model to weights-improvement-05-6.0868.hdf5\n",
      "26594/26594 [==============================] - 142s - loss: 6.0868   \n",
      "Epoch 7/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 6.0396Epoch 00006: loss improved from 6.08678 to 6.03968, saving model to weights-improvement-06-6.0397.hdf5\n",
      "26594/26594 [==============================] - 144s - loss: 6.0397   \n",
      "Epoch 8/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 5.9852Epoch 00007: loss improved from 6.03968 to 5.98450, saving model to weights-improvement-07-5.9845.hdf5\n",
      "26594/26594 [==============================] - 142s - loss: 5.9845   \n",
      "Epoch 9/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 5.9246Epoch 00008: loss improved from 5.98450 to 5.92684, saving model to weights-improvement-08-5.9268.hdf5\n",
      "26594/26594 [==============================] - 143s - loss: 5.9268   \n",
      "Epoch 10/10\n",
      "26496/26594 [============================>.] - ETA: 0s - loss: 5.8584Epoch 00009: loss improved from 5.92684 to 5.85827, saving model to weights-improvement-09-5.8583.hdf5\n",
      "26594/26594 [==============================] - 142s - loss: 5.8583   \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xcdb000b8>"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, Y_train, nb_epoch=10, batch_size=128, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# load the network weights\n",
    "filename = \"data/weights/weights-improvement-18-2.8349.hdf5\"\n",
    "model.load_weights(filename)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.9220Epoch 00000: loss improved from inf to 2.92197, saving model to data/weights/word-weights-improvement-00-2.9220.hdf5\n",
      "26594/26594 [==============================] - 213s - loss: 2.9220   \n",
      "Epoch 2/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.8963Epoch 00001: loss improved from 2.92197 to 2.89641, saving model to data/weights/word-weights-improvement-01-2.8964.hdf5\n",
      "26594/26594 [==============================] - 213s - loss: 2.8964   \n",
      "Epoch 3/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.0207Epoch 00002: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 4.0208   \n",
      "Epoch 4/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.3711Epoch 00003: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 6.3712   \n",
      "Epoch 5/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.8421Epoch 00004: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 5.8421   \n",
      "Epoch 6/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.4544Epoch 00005: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 5.4543   \n",
      "Epoch 7/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.2229Epoch 00006: loss did not improve\n",
      "26594/26594 [==============================] - 214s - loss: 5.2228   \n",
      "Epoch 8/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.5285Epoch 00007: loss did not improve\n",
      "26594/26594 [==============================] - 214s - loss: 4.5284   \n",
      "Epoch 9/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.5803Epoch 00008: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 4.5802   \n",
      "Epoch 10/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.9313Epoch 00009: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.9312   \n",
      "Epoch 11/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.6743Epoch 00010: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.6743   \n",
      "Epoch 12/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.5347Epoch 00011: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.5348   \n",
      "Epoch 13/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.5881Epoch 00012: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.5882   \n",
      "Epoch 14/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.2959Epoch 00013: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.2959   \n",
      "Epoch 15/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.1784Epoch 00014: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.1785   \n",
      "Epoch 16/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.3358Epoch 00015: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.3357   \n",
      "Epoch 17/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.2340Epoch 00016: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.2339   \n",
      "Epoch 18/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.0942Epoch 00017: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.0941   \n",
      "Epoch 19/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.9974Epoch 00018: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 2.9975   \n",
      "Epoch 20/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.0067Epoch 00019: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.0068   \n",
      "Epoch 21/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.8871Epoch 00020: loss improved from 2.89641 to 2.88717, saving model to data/weights/word-weights-improvement-20-2.8872.hdf5\n",
      "26594/26594 [==============================] - 213s - loss: 2.8872   \n",
      "Epoch 22/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 2.9705Epoch 00021: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 2.9704   \n",
      "Epoch 23/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.7911Epoch 00022: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.7913   \n",
      "Epoch 24/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.5953Epoch 00023: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 6.5956   \n",
      "Epoch 25/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.8460Epoch 00024: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 5.8461   \n",
      "Epoch 26/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.2479Epoch 00025: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 5.2478   \n",
      "Epoch 27/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.5419Epoch 00026: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 4.5419   \n",
      "Epoch 28/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.8770Epoch 00027: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.8771   \n",
      "Epoch 29/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.4541Epoch 00028: loss did not improve\n",
      "26594/26594 [==============================] - 214s - loss: 3.4541   \n",
      "Epoch 30/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.2267Epoch 00029: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.2267   \n",
      "Epoch 31/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.1072Epoch 00030: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.1073   \n",
      "Epoch 32/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.4198Epoch 00031: loss did not improve\n",
      "26594/26594 [==============================] - 214s - loss: 3.4200   \n",
      "Epoch 33/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 7.0286Epoch 00032: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 7.0285   \n",
      "Epoch 34/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.1375Epoch 00033: loss did not improve\n",
      "26594/26594 [==============================] - 214s - loss: 6.1374   \n",
      "Epoch 35/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.8121Epoch 00034: loss did not improve\n",
      "26594/26594 [==============================] - 214s - loss: 5.8121   \n",
      "Epoch 36/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.5293Epoch 00035: loss did not improve\n",
      "26594/26594 [==============================] - 214s - loss: 5.5293   \n",
      "Epoch 37/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.1225Epoch 00036: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 5.1223   \n",
      "Epoch 38/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.5301Epoch 00037: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 4.5301   \n",
      "Epoch 39/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.0578Epoch 00038: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 4.0581   \n",
      "Epoch 40/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.0609Epoch 00039: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 6.0612   \n",
      "Epoch 41/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.1230Epoch 00040: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 6.1230   \n",
      "Epoch 42/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.7685Epoch 00041: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 5.7684   \n",
      "Epoch 43/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.5108Epoch 00042: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 5.5107   \n",
      "Epoch 44/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.1861Epoch 00043: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 5.1861   \n",
      "Epoch 45/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.7884Epoch 00044: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 4.7884   \n",
      "Epoch 46/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.3749Epoch 00045: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 4.3747   \n",
      "Epoch 47/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.0282Epoch 00046: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 4.0282   \n",
      "Epoch 48/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.8594Epoch 00047: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.8595   \n",
      "Epoch 49/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.7856Epoch 00048: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.7855   \n",
      "Epoch 50/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.5006Epoch 00049: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.5005   \n",
      "Epoch 51/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.5421Epoch 00050: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.5422   \n",
      "Epoch 52/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.5264Epoch 00051: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.5266   \n",
      "Epoch 53/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.4776Epoch 00052: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.4775   \n",
      "Epoch 54/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.3556Epoch 00053: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.3556   \n",
      "Epoch 55/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.2450Epoch 00054: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.2452   \n",
      "Epoch 56/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.1577Epoch 00055: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.1578   \n",
      "Epoch 57/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.3940Epoch 00056: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.3939   \n",
      "Epoch 58/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.3572Epoch 00057: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.3573   \n",
      "Epoch 59/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.2033Epoch 00058: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.2035   \n",
      "Epoch 60/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.1477Epoch 00059: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.1475   \n",
      "Epoch 61/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.3988Epoch 00060: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 3.3990   \n",
      "Epoch 62/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.1584Epoch 00061: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 6.1583   \n",
      "Epoch 63/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.7129Epoch 00062: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 6.7129   \n",
      "Epoch 64/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.9968Epoch 00063: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 5.9969   \n",
      "Epoch 65/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.4649Epoch 00064: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 6.4648   \n",
      "Epoch 66/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 6.1373Epoch 00065: loss did not improve\n",
      "26594/26594 [==============================] - 214s - loss: 6.1375   \n",
      "Epoch 67/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 5.5118Epoch 00066: loss did not improve\n",
      "26594/26594 [==============================] - 215s - loss: 5.5119   \n",
      "Epoch 68/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.9950Epoch 00067: loss did not improve\n",
      "26594/26594 [==============================] - 213s - loss: 4.9951   \n",
      "Epoch 69/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.6731Epoch 00068: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 4.6730   \n",
      "Epoch 70/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.4132Epoch 00069: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 4.4134   \n",
      "Epoch 71/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.1735Epoch 00070: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 4.1736   \n",
      "Epoch 72/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 4.0028Epoch 00071: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 4.0027   \n",
      "Epoch 73/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.9368Epoch 00072: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.9368   \n",
      "Epoch 74/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.8320Epoch 00073: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.8320   \n",
      "Epoch 75/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.7297Epoch 00074: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.7298   \n",
      "Epoch 76/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.7951Epoch 00075: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.7952   \n",
      "Epoch 77/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.7333Epoch 00076: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.7332   \n",
      "Epoch 78/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.6085Epoch 00077: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.6085   \n",
      "Epoch 79/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.5408Epoch 00078: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.5408   \n",
      "Epoch 80/80\n",
      "26592/26594 [============================>.] - ETA: 0s - loss: 3.5464Epoch 00079: loss did not improve\n",
      "26594/26594 [==============================] - 212s - loss: 3.5464   \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xc175b160>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, Y_train, nb_epoch=80, batch_size=32, callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load the network weights\n",
    "filename = \"weights-improvement-18-2.8349.hdf5\"\n",
    "model.load_weights(filename)\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed:\n",
      "\" he had to ask his neighbour to tell him A nice muddle their slates'll be in before the trial's over thought Alice One of the jurors had a pencil that squeaked This of course Alice could not stand and she went round the court and got behind him and very soon found an opportunity of taking it away She did it so quickly that the poor little juror it was Bill the Lizard could not make out at all what had become of it so after hunting all about for it he was obliged to write with one finger for \"\n",
      "\n",
      "Generated Sequence:\n",
      "the rest the the the and it that of the the and talking I she I the understand to the the the happen key key the spell Look and she So shan't the would of at this Footman's upon I she to to the that to pebbles the the the the dance Suppose you Alice's it it May who much of thought Alice in a perhaps half and so to to to to pebbles the the it the dance Will you won't you will you won't won't you you join join dance Thank I I think THAT said Duchess said the Alice said truthful Alice said You've the could on the YOURS and the to whiting to that down the a high of the fact the speaking the dark and and plate the the the W the Our it I'll and the to the the to to thing it the the the the you dance Will you won't you will you won't won't won't join join the dance You can won't think you you Hatter couple a it tone to She thought a tis off nose the the the the the the the you teacups it wept court it the\n",
      "\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "start = np.random.randint(0, len(X_train)-1)\n",
    "pattern = dataX[start]\n",
    "result = []\n",
    "print \"Seed:\"\n",
    "print \"\\\"\", ' '.join([int_to_word[value] for value in pattern]), \"\\\"\"\n",
    "for i in xrange(200):\n",
    "    x = np.reshape(pattern, (1, len(pattern), 1))\n",
    "    x = x/float(n_vocab)\n",
    "    prediction = model.predict(x)\n",
    "    index = np.argmax(prediction)\n",
    "    result.append(int_to_word[index])\n",
    "    pattern.append(index)\n",
    "    pattern = pattern[1:len(pattern)]\n",
    "print \"\\nGenerated Sequence:\"\n",
    "print ' '.join(result)\n",
    "print \"\\nDone.\"  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}