charLSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import time\n",
    "import random\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR='data/shakespeare.txt'\n",
    "BATCH_SIZE=25\n",
    "LAYER_NUM=1\n",
    "SEQ_LENGTH=50\n",
    "HIDDEN_DIM=128\n",
    "GENERATE_LEN=500\n",
    "NB_EPOCH=1500\n",
    "MODE='train'\n",
    "WEIGHTS=''\n",
    "LEARNING_RATE=1e-1\n",
    "IS_TRAIN=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_text_data():\n",
    "    with open(DATA_DIR,\"r\") as text_file:\n",
    "        data=text_file.read()\n",
    "        tr_data,va_data=data[0:1070392],data[1070392:1075392]\n",
    "    lis=list(set(data))    \n",
    "    total_char,unique_char=len(data),len(lis)\n",
    "    print ('data has %d characters, %d unique.' %(total_char, unique_char))\n",
    "    char_to_ix = { ch:i for i,ch in enumerate(lis) }\n",
    "    ix_to_char = { i:ch for i,ch in enumerate(lis) }\n",
    "#     return total_char,unique_char,char_to_ix,ix_to_char,data,tr_data,va_data\n",
    "    return data\n",
    "\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data has 1115393 characters, 65 unique.\n"
     ]
    }
   ],
   "source": [
    "data=get_text_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/coding2.json') as file:\n",
    "    da=json.load(file)\n",
    "    total_chars,unique_chars,char_id=da['total'],da['unique'],da['char_id']\n",
    "\n",
    "id_char={int(v):k for k,v in char_id.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_next_batch(given_data, seq_length, p):\n",
    "    inputs = [char_id[ch] for ch in given_data[p:p+seq_length]]\n",
    "    targets = [char_id[ch] for ch in given_data[p+1:p+seq_length+1]]\n",
    "    inputs=np.array(inputs).reshape(1,seq_length)\n",
    "    targets=np.array(targets).reshape(1,seq_length)\n",
    "    return inputs,targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Preprocessing- generated all the batches at once rather than dynamic generation\n",
    "ptr=0\n",
    "batches=(total_chars//SEQ_LENGTH)\n",
    "x=[]\n",
    "y=[]\n",
    "for batch in range(batches):\n",
    "    a,b=generate_next_batch(data, SEQ_LENGTH, ptr)\n",
    "    x.append(a)\n",
    "    y.append(b)\n",
    "    ptr+=SEQ_LENGTH\n",
    "x=np.array(x).reshape(batches, SEQ_LENGTH)\n",
    "y=np.array(y).reshape(batches, SEQ_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22307, 50)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample(n):\n",
    "        seed_int=random.randint(0,64)\n",
    "        seed_seq=id_char[seed_int]\n",
    "        model2 = CharRNN(unique_chars, sampling=True,\n",
    "                    hidden_dim=HIDDEN_DIM, num_layers=LAYER_NUM,\n",
    "                    learning_rate=LEARNING_RATE)\n",
    "        checkpoint_path=tf.train.latest_checkpoint('checkpoint/')\n",
    "        with tf.Session() as sess:\n",
    "            saver=tf.train.Saver()\n",
    "            saver.restore(sess, checkpoint_path)\n",
    "            init_state=sess.run(model2.initial_state)\n",
    "            xx=np.zeros((1,1))\n",
    "            ixes=[]\n",
    "            xx[0][0]=char_id[seed_seq]\n",
    "            ixes.append(char_id[seed_seq])\n",
    "            for t in range(n):\n",
    "                prob,init_state=sess.run([model2.prediction,model2.final_state],\n",
    "                                         feed_dict={model2.inputs:xx, model2.initial_state:init_state})\n",
    "                ix=np.random.choice(range(unique_chars), p=prob.ravel())\n",
    "                ixes.append(ix)\n",
    "                xx[0][0]=ix\n",
    "            return ''.join(id_char[ch] for ch in ixes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_LSTM(hidden_dim, num_layers, batch_size):\n",
    "    with tf.name_scope('lstm'):\n",
    "        cells= [tf.contrib.rnn.BasicLSTMCell(num_units=HIDDEN_DIM) for layer in range(LAYER_NUM)]\n",
    "        multi_layer_cell = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True)\n",
    "        _initial_state =  multi_layer_cell.zero_state(batch_size, tf.float32)\n",
    "        return multi_layer_cell, _initial_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def output(lstm_output, hidden_dim, out_size):\n",
    "\n",
    "    seq_output = tf.concat(lstm_output, axis=1)\n",
    "    x = tf.reshape(seq_output, [-1, hidden_dim])\n",
    "    \n",
    "    with tf.variable_scope('softmax'):\n",
    "        softmax_w = tf.Variable(tf.truncated_normal([hidden_dim, out_size],stddev=0.1))\n",
    "        softmax_b = tf.Variable(tf.zeros([out_size]))\n",
    "    \n",
    "    logits = tf.matmul(x, softmax_w) + softmax_b\n",
    "    \n",
    "    out = tf.nn.softmax(logits, name='predictions')\n",
    "    \n",
    "    return out, logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss(logits, targets, lstm_size, vocab_size):\n",
    "    \n",
    "    y_one_hot = tf.one_hot(targets, vocab_size)\n",
    "    y_reshaped = tf.reshape(y_one_hot, logits.get_shape())\n",
    "    \n",
    "    # Softmax cross entropy loss\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_reshaped, logits=logits))\n",
    "    \n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimizer(loss, learning_rate, grad_clip=5):\n",
    "    \n",
    "    # Optimizer for training, using gradient clipping to control exploding gradients\n",
    "    tvars = tf.trainable_variables()\n",
    "    grads, _ = tf.clip_by_global_norm(tf.gradients(loss, tvars), grad_clip)\n",
    "    train_op = tf.train.AdamOptimizer(learning_rate)\n",
    "    optimizer = train_op.apply_gradients(zip(grads, tvars))\n",
    "    \n",
    "    return optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CharRNN:\n",
    "    \n",
    "    def __init__(self, vocab_size, batch_size=25, seq_length=50, hidden_dim=128, num_layers=2, \n",
    "                 learning_rate=0.001, grad_clip=5, sampling=False):\n",
    "    \n",
    "        if sampling == True:\n",
    "            batch_size, seq_length = 1, 1\n",
    "        else:\n",
    "            batch_size, seq_length = batch_size, seq_length\n",
    "        \n",
    "        self.inputs, self.targets = tf.placeholder(tf.int32,[batch_size,seq_length],name='inputs'), tf.placeholder(tf.int32,[batch_size, seq_length],name='targets')\n",
    "\n",
    "        # Build the LSTM cell\n",
    "        multi_cell, self.initial_state = build_LSTM(hidden_dim, num_layers=num_layers, batch_size=batch_size)\n",
    "        \n",
    "        x_one_hot = tf.one_hot(self.inputs, vocab_size)\n",
    "        \n",
    "        outputs, state = tf.nn.dynamic_rnn(multi_cell, x_one_hot, initial_state=self.initial_state)\n",
    "        self.final_state = state\n",
    "        \n",
    "        self.prediction, self.logits = output(outputs, hidden_dim, vocab_size)\n",
    "        \n",
    "        # Loss and optimizer (with gradient clipping)\n",
    "        self.loss = loss(self.logits, self.targets, hidden_dim,vocab_size)\n",
    "        self.optimizer = optimizer(self.loss, learning_rate, grad_clip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_of_batches=int(len(x)//BATCH_SIZE)\n",
    "ptr=0\n",
    "iteration=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1/100...  Training Step: 0...  Training loss: 4.175072...  0.1027 sec/batch\n",
      "Epoch: 1/100...  Training Step: 100...  Training loss: 2.154204...  0.0638 sec/batch\n",
      "Epoch: 1/100...  Training Step: 200...  Training loss: 2.031109...  0.0638 sec/batch\n",
      "Epoch: 1/100...  Training Step: 300...  Training loss: 1.932110...  0.0638 sec/batch\n",
      "Epoch: 1/100...  Training Step: 400...  Training loss: 1.856728...  0.0688 sec/batch\n",
      "Epoch: 1/100...  Training Step: 500...  Training loss: 1.917502...  0.0638 sec/batch\n",
      "Epoch: 1/100...  Training Step: 600...  Training loss: 1.789049...  0.0648 sec/batch\n",
      "Epoch: 1/100...  Training Step: 700...  Training loss: 1.754209...  0.0678 sec/batch\n",
      "Epoch: 1/100...  Training Step: 800...  Training loss: 1.644505...  0.0678 sec/batch\n",
      "Epoch: 2/100...  Training Step: 900...  Training loss: 2.074351...  0.0648 sec/batch\n",
      "Epoch: 2/100...  Training Step: 1000...  Training loss: 1.909281...  0.0648 sec/batch\n",
      "Epoch: 2/100...  Training Step: 1100...  Training loss: 1.694085...  0.0638 sec/batch\n",
      "Epoch: 2/100...  Training Step: 1200...  Training loss: 1.781513...  0.0658 sec/batch\n",
      "Epoch: 2/100...  Training Step: 1300...  Training loss: 1.774521...  0.0658 sec/batch\n",
      "Epoch: 2/100...  Training Step: 1400...  Training loss: 1.724285...  0.0648 sec/batch\n",
      "Epoch: 2/100...  Training Step: 1500...  Training loss: 1.808253...  0.0668 sec/batch\n",
      "Epoch: 2/100...  Training Step: 1600...  Training loss: 1.767091...  0.0638 sec/batch\n",
      "Epoch: 2/100...  Training Step: 1700...  Training loss: 1.698340...  0.0648 sec/batch\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-105-7eacfde53e1c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     26\u001b[0m                                                  \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfinal_state\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m                                                  model.optimizer], \n\u001b[1;32m---> 28\u001b[1;33m                                                  feed_dict=feed)\n\u001b[0m\u001b[0;32m     29\u001b[0m             \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     30\u001b[0m             \u001b[1;32mif\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miteration\u001b[0m\u001b[1;33m%\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m==\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\anaconda\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    898\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    899\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 900\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    901\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    902\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\anaconda\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1133\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1134\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1135\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1136\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1137\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\anaconda\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1314\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1315\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[1;32m-> 1316\u001b[1;33m                            run_metadata)\n\u001b[0m\u001b[0;32m   1317\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1318\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\anaconda\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1320\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1321\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1322\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1323\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1324\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\anaconda\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1305\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1306\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m-> 1307\u001b[1;33m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[0;32m   1308\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1309\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32md:\\anaconda\\envs\\tensorflow-gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   1407\u001b[0m       return tf_session.TF_SessionRun_wrapper(\n\u001b[0;32m   1408\u001b[0m           \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1409\u001b[1;33m           run_metadata)\n\u001b[0m\u001b[0;32m   1410\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1411\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraise_exception_on_not_ok_status\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mstatus\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "epochs = 100\n",
    " # Save every N iterations# Save e \n",
    "save_every_n = 200\n",
    "\n",
    "model = CharRNN(unique_chars, batch_size=BATCH_SIZE, seq_length=SEQ_LENGTH,\n",
    "                hidden_dim=HIDDEN_DIM, num_layers=LAYER_NUM, \n",
    "                learning_rate=LEARNING_RATE)\n",
    "\n",
    "saver = tf.train.Saver(max_to_keep=100)\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    zero_state=sess.run(model.initial_state)\n",
    "    for e in range(epochs):\n",
    "        new_state = zero_state\n",
    "        for i in range(no_of_batches):\n",
    "            if (ptr+BATCH_SIZE+1 >= batches): \n",
    "                ptr=0\n",
    "            inputs,targets=x[ptr:ptr+BATCH_SIZE],y[ptr:ptr+BATCH_SIZE]\n",
    "            ptr+=BATCH_SIZE\n",
    "            start=time.time()\n",
    "            feed = {model.inputs: inputs,\n",
    "                    model.targets: targets,\n",
    "                    model.initial_state: new_state}\n",
    "            batch_loss, new_state, _ = sess.run([model.loss, \n",
    "                                                 model.final_state, \n",
    "                                                 model.optimizer], \n",
    "                                                 feed_dict=feed)\n",
    "            end = time.time()\n",
    "            if(iteration%100==0):\n",
    "                print('Epoch: %i/%i... '%(e+1, epochs),\n",
    "                      'Training Step: %i... '%iteration,\n",
    "                      'Training loss: %f... '%batch_loss,\n",
    "                      '{:.4f} sec/batch'.format(end-start))\n",
    "            iteration+=1\n",
    "        if (e % 2 == 0):\n",
    "            saver.save(sess, \"checkk/i{}_l{}.ckpt\".format(iteration//no_of_batches, batch_loss))\n",
    "    saver.save(sess, \"checkk/i{}_l{}.ckpt\".format(counter, HIDDEN_DIM))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from checkpoint/i43708_l1.4605046510696411.ckpt\n",
      "bel intoer full\n",
      "not a match'd me.\n",
      "Much eaten, in Talk, sharm to theirs! afier,\n",
      "To r my brother be tamas\n",
      "Are dint here adred\n",
      "for hast will beer sinming him us, a greate and for\n",
      "a glasse as it\n",
      "me, they d\n"
     ]
    }
   ],
   "source": [
    "reset_graph()\n",
    "print(sample(200))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}