autoencoder_lstm_one_hot.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Autoencoder Encoder-Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "random.seed(1337)\n",
    "\n",
    "class EncoderDecoder:\n",
    "    def __init__(self, vocabulary={}, state_size=64, n_max_length=30):     \n",
    "        self.state_size = state_size\n",
    "        self.n_max_length = n_max_length\n",
    "        self.vocabulary = vocabulary\n",
    "\n",
    "        \n",
    "        ######################\n",
    "        # Graph Construction #\n",
    "        ######################\n",
    "        self.graph = tf.Graph()\n",
    "        with self.graph.as_default():\n",
    "            self.sen_en = tf.placeholder(tf.int32, shape=(None, self.n_max_length), name=\"sen_en\")\n",
    "            self.sen_de = tf.placeholder(tf.int32, shape=(None, self.n_max_length), name=\"sen_de\")\n",
    "            self.sen_en_length = tf.placeholder(tf.int32, shape=(None,), name=\"sen_en_length\")\n",
    "            self.sen_de_length = tf.placeholder(tf.int32, shape=(None,), name=\"sen_de_length\")\n",
    "            \n",
    "            batch_size = tf.shape(self.sen_en)[0]\n",
    "            \n",
    "            # TODO sen_en_embedding could also be self-trained embedding: embedding_lookup\n",
    "            self.sen_en_embedding = tf.one_hot(self.sen_en, len(self.vocabulary))\n",
    "            self.sen_de_embedding = tf.one_hot(self.sen_de, len(self.vocabulary))\n",
    "            \n",
    "            # build encoder decoder structure\n",
    "            with tf.variable_scope(\"encoder\") as scope:\n",
    "                self.cell_en = tf.contrib.rnn.BasicLSTMCell(self.state_size)\n",
    "            with tf.variable_scope(\"decoder\") as scope:\n",
    "                self.cell_de = tf.contrib.rnn.BasicLSTMCell(self.state_size)\n",
    "            with tf.variable_scope(\"encoder\") as scope:\n",
    "                self.cell_en_init = self.cell_en.zero_state(batch_size, tf.float32)\n",
    "                self.h_state_en, self.final_state_en = tf.nn.dynamic_rnn(\n",
    "                    self.cell_en,\n",
    "                    self.sen_en_embedding,\n",
    "                    sequence_length=self.sen_en_length,\n",
    "                    initial_state=self.cell_en_init,\n",
    "                )\n",
    "            with tf.variable_scope(\"decoder\") as scope:\n",
    "                self.cell_de_init = self.final_state_en\n",
    "                self.h_state_de, self.final_state_de = tf.nn.dynamic_rnn(\n",
    "                    self.cell_de,\n",
    "                    self.sen_de_embedding,\n",
    "                    sequence_length=self.sen_de_length,\n",
    "                    initial_state=self.cell_de_init,\n",
    "                )\n",
    "            \n",
    "\n",
    "            with tf.variable_scope(\"softmax\") as scope:\n",
    "                W = tf.get_variable(\"W\", [self.state_size, len(self.vocabulary)], initializer=tf.random_normal_initializer(seed=None))\n",
    "                b = tf.get_variable(\"b\", [len(self.vocabulary)], initializer=tf.random_normal_initializer(seed=None))               \n",
    "            self.logits = tf.reshape(\n",
    "                tf.add(tf.matmul(tf.reshape(self.h_state_de, (-1, self.state_size)), W), b),\n",
    "                shape=(-1, self.n_max_length, len(self.vocabulary))\n",
    "            )\n",
    "            self.prediction = tf.nn.softmax(self.logits)\n",
    "                \n",
    "            # construct loss and train op\n",
    "            self.cross_ent = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                labels=self.sen_en,\n",
    "                logits=self.logits\n",
    "            )        \n",
    "            #self.mask = tf.sign(tf.reduce_max(tf.abs(self.sen_de_embedding), 2))\n",
    "            self.mask = tf.sequence_mask(self.sen_de_length, maxlen=self.n_max_length)\n",
    "            self.loss = tf.reduce_mean(\n",
    "                #tf.reduce_sum(tf.multiply(self.cross_ent, self.mask), 1) / tf.reduce_sum(self.mask, 1)\n",
    "                tf.divide(\n",
    "                    tf.reduce_sum(\n",
    "                        tf.where(\n",
    "                            self.mask,\n",
    "                            self.cross_ent,\n",
    "                            tf.zeros_like(self.cross_ent)\n",
    "                        ), 1\n",
    "                    ),\n",
    "                    tf.to_float(self.sen_de_length)\n",
    "                )\n",
    "            )\n",
    "            \n",
    "            \"\"\"\n",
    "            optimizer = tf.train.AdamOptimizer()\n",
    "            self.op_train = optimizer.minimize(self.loss)\n",
    "            \"\"\"\n",
    "            # Calculate and clip gradients\n",
    "            params = tf.trainable_variables()\n",
    "            gradients = tf.gradients(self.loss, params)\n",
    "            self.clipped_gradients, _ = tf.clip_by_global_norm(gradients, 1)\n",
    "            # Optimization\n",
    "            optimizer = tf.train.AdamOptimizer()\n",
    "            self.op_train = optimizer.apply_gradients(zip(self.clipped_gradients, params))\n",
    "            \n",
    "            # initializer\n",
    "            gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.1)\n",
    "            self.sess = tf.Session(\n",
    "                graph=self.graph,\n",
    "                config=tf.ConfigProto(gpu_options=gpu_options)\n",
    "            )           \n",
    "            self.init = tf.global_variables_initializer()\n",
    "            self.sess.run(self.init)\n",
    "            \n",
    "    def train(self, batch_sen_en, batch_sen_de, batch_sen_en_length, batch_sen_de_length):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        batch_sen_en: numpy, shape=(n, max_length), dtype=int\n",
    "        batch_sen_de: numpy, shape=(n, max_length), dtype=int\n",
    "        batch_sen_en_length: numpy, shape=(n,), dtype=int\n",
    "        batch_sen_de_length: numpy, shape=(n,), dtype=int\n",
    "        \"\"\"\n",
    "        assert batch_sen_en.shape[0] == batch_sen_de.shape[0]\n",
    "        _, loss, prediction, sen_en_embedding, mask, cross_ent, clipped_gradients = self.sess.run(\n",
    "            [self.op_train, self.loss, self.prediction, self.sen_en_embedding, self.mask, self.cross_ent, self.clipped_gradients],\n",
    "            feed_dict={\n",
    "                self.sen_en: batch_sen_en,\n",
    "                self.sen_de: batch_sen_de,\n",
    "                self.sen_en_length: batch_sen_en_length,\n",
    "                self.sen_de_length: batch_sen_de_length,\n",
    "            }\n",
    "        )\n",
    "        return loss, prediction, sen_en_embedding, mask, cross_ent, clipped_gradients\n",
    "        \n",
    "    def predict(self, batch_sen_en, batch_sen_de, batch_sen_en_length, batch_sen_de_length):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        batch_sen_en: numpy, shape=(n, max_length), dtype=int\n",
    "        batch_sen_de: numpy, shape=(n, max_length), dtype=int\n",
    "        batch_sen_en_length: numpy, shape=(n,), dtype=int\n",
    "        batch_sen_de_length: numpy, shape=(n,), dtype=int\n",
    "        \"\"\"\n",
    "        assert batch_sen_en.shape[0] == batch_sen_de.shape[0]\n",
    "        loss, prediction = self.sess.run(\n",
    "            [self.loss, self.prediction],\n",
    "            feed_dict={\n",
    "                self.sen_en: batch_sen_en,\n",
    "                self.sen_de: batch_sen_de,\n",
    "                self.sen_en_length: batch_sen_en_length,\n",
    "                self.sen_de_length: batch_sen_de_length,\n",
    "            }\n",
    "        )\n",
    "        return loss, prediction\n",
    "\n",
    "    \n",
    "def evaluate(batch_sen_en, batch_sen_en_length, batch_prediction, vocabulary):\n",
    "    \"\"\"\n",
    "    Parameters\n",
    "    ----------\n",
    "    batch_sen_en: numpy, shape=(n, max_length), dtype=int\n",
    "    batch_sen_en_length: numpy, shape=(n,), dtype=int\n",
    "    batch_prediction: numpy, shape=(n, max_length, len(vocabulary))\n",
    "    \"\"\"\n",
    "    assert batch_sen_en.shape[0] == batch_prediction.shape[0]\n",
    "    acc_word = 0\n",
    "    acc_sen_end = 0\n",
    "    for i in range(batch_sen_en.shape[0]):\n",
    "        is_first_end = False\n",
    "        for j in range(batch_sen_en_length[i]):\n",
    "            cur_pred_word = np.argmax(batch_prediction[i, j])\n",
    "            if cur_pred_word == batch_sen_en[i, j]:\n",
    "                acc_word += 1\n",
    "                if not is_first_end and cur_pred_word == vocabulary[\"</s>\"]:\n",
    "                    acc_sen_end += 1\n",
    "            if cur_pred_word == vocabulary[\"</s>\"]:\n",
    "                is_first_end = True\n",
    "    return 1. * acc_word / np.sum(batch_sen_en_length), 1. * acc_sen_end / batch_sen_en.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[109 142  72 196 164  88  36  51 150 147  49 115  19  34 112  48  44  68\n",
      "  72   5  57  49  95 104 203 200 200 200 200 200]\n",
      "[ 65 164 191  17  66 142  54  43 140 192 197 105 168 143 169  33 180  19\n",
      "  52 196 193   6  54  14 183 203 200 200 200 200]\n",
      "epoch 0 last batch (0.052609067579127457, 0.0)\n",
      "epoch 1 last batch (0.064157399486740804, 0.0)\n",
      "epoch 2 last batch (0.088109495295124032, 0.23)\n",
      "epoch 3 last batch (0.094097519247219846, 0.24)\n",
      "epoch 4 last batch (0.11291702309666382, 0.59)\n",
      "epoch 5 last batch (0.12917023096663816, 0.76)\n",
      "epoch 6 last batch (0.12745936698032506, 0.47)\n",
      "epoch 7 last batch (0.14499572284003423, 0.71)\n",
      "epoch 8 last batch (0.14414029084687768, 0.85)\n",
      "epoch 9 last batch (0.16295979469632163, 0.66)\n",
      "epoch 10 last batch (0.17108639863130881, 0.78)\n",
      "epoch 11 last batch (0.16680923866552608, 0.77)\n",
      "epoch 12 last batch (0.1775021385799829, 0.76)\n",
      "epoch 13 last batch (0.18092386655260906, 0.65)\n",
      "epoch 14 last batch (0.19289991445680069, 0.63)\n",
      "epoch 15 last batch (0.18776732249786143, 0.59)\n",
      "epoch 16 last batch (0.20615911035072712, 0.68)\n",
      "epoch 17 last batch (0.20829769033361847, 0.5)\n",
      "epoch 18 last batch (0.22455089820359281, 0.64)\n",
      "epoch 19 last batch (0.23567151411462789, 0.74)\n",
      "train (0.24877776256094883, 0.9109666666666667)\n",
      "test (0.24410132983460697, 0.9111)\n"
     ]
    }
   ],
   "source": [
    "def generate_data(n, max_length, origin_vocabulary, extend_vocabulary):\n",
    "    sen_en = np.full((n, max_length), extend_vocabulary[\"<pad>\"], dtype=np.int32)\n",
    "    sen_de = np.full((n, max_length), extend_vocabulary[\"<pad>\"], dtype=np.int32)\n",
    "    sen_en_length = np.zeros((n,), dtype=np.int32)\n",
    "    sen_de_length = np.zeros((n,), dtype=np.int32)\n",
    "\n",
    "    def get_random_sequence(length, max_length):\n",
    "        x = np.full((max_length), extend_vocabulary[\"<pad>\"], dtype=np.int32)\n",
    "        for i in range(length):\n",
    "            x[i] = extend_vocabulary[random.choice(list(origin_vocabulary))]\n",
    "        return x\n",
    "\n",
    "    for i in range(n):\n",
    "        l = random.randint(max_length // 2, max_length-1)\n",
    "        sen_en[i, :] = get_random_sequence(l, max_length)\n",
    "        sen_en[i, l] = extend_vocabulary[\"</s>\"]\n",
    "        sen_de[i, 1:l+1] = sen_en[i, :l]\n",
    "        sen_de[i, 0] = extend_vocabulary[\"<s>\"]\n",
    "        sen_en_length[i] = l + 1\n",
    "        sen_de_length[i] = l + 1\n",
    "    \n",
    "    return sen_en, sen_de, sen_en_length, sen_de_length\n",
    "\n",
    "def get_total_accuracy(data_sen_en, data_sen_de, data_sen_en_length, data_sen_de_length, extend_vocabulary):\n",
    "    n_hit_word, n_hit_length = 0, 0\n",
    "    n_total_word, n_total_length = 0, 0\n",
    "    cur_idx = 0\n",
    "    while cur_idx < data_sen_en.shape[0]:\n",
    "        batch_sen_en = data_sen_en[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_de = data_sen_de[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_en_length = data_sen_en_length[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_de_length = data_sen_de_length[cur_idx: cur_idx + n_batch_size]\n",
    "        \n",
    "        _, predictions = pretrained_lstm.predict(\n",
    "            batch_sen_en, batch_sen_de, batch_sen_en_length, batch_sen_de_length\n",
    "        )\n",
    "        cur_idx += n_batch_size\n",
    "        cur_acc_word, cur_acc_length = evaluate(batch_sen_en, batch_sen_en_length, predictions, extend_vocabulary)\n",
    "        n_hit_word += cur_acc_word * np.sum(batch_sen_en_length)\n",
    "        n_total_word += np.sum(batch_sen_en_length)\n",
    "        n_hit_length += cur_acc_length * batch_sen_en.shape[0]\n",
    "        n_total_length += batch_sen_en.shape[0]\n",
    "    return 1. * n_hit_word / n_total_word, 1. * n_hit_length / n_total_length\n",
    "    \n",
    "# hyperparameter\n",
    "vocabulary_size = 200\n",
    "origin_vocabulary = {}\n",
    "for i in range(vocabulary_size):\n",
    "    origin_vocabulary[\"{}\".format(i)] = len(origin_vocabulary)\n",
    "extend_vocabulary = dict(origin_vocabulary)\n",
    "for w in [\"<pad>\", \"<unk>\", \"<s>\", \"</s>\"]:\n",
    "    extend_vocabulary[w] = len(extend_vocabulary)\n",
    "#vocabulary = {\"<pad>\": 0, \"<unk>\": 1, \"<s>\": 2, \"</s>\": 3, \"a\": 4, \"b\": 5}\n",
    "state_size=64\n",
    "n_max_length=30\n",
    "n_batch_size=100\n",
    "\n",
    "# generate training/testing data\n",
    "n_train = 60000\n",
    "n_test = 10000\n",
    "train_sen_en, train_sen_de, train_sen_en_length, train_sen_de_length = generate_data(n_train, n_max_length,\n",
    "                                                                                     origin_vocabulary, extend_vocabulary)\n",
    "test_sen_en, test_sen_de, test_sen_en_length, test_sen_de_length = generate_data(n_test, n_max_length,\n",
    "                                                                                 origin_vocabulary, extend_vocabulary)\n",
    "print(train_sen_en[0])\n",
    "print(test_sen_en[0])\n",
    "\n",
    "pretrained_lstm = EncoderDecoder(vocabulary=extend_vocabulary, state_size=state_size, n_max_length=n_max_length)\n",
    "\n",
    "for epoch in range(20):\n",
    "    cur_idx = 0\n",
    "    while cur_idx < train_sen_en.shape[0]:\n",
    "        batch_sen_en = train_sen_en[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_de = train_sen_de[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_en_length = train_sen_en_length[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_de_length = train_sen_de_length[cur_idx: cur_idx + n_batch_size]\n",
    "        \n",
    "        loss, predictions, sen_en_embedding, mask, cross_ent, clipped_gradients = pretrained_lstm.train(\n",
    "            batch_sen_en, batch_sen_de, batch_sen_en_length, batch_sen_de_length\n",
    "        )\n",
    "        cur_idx += n_batch_size\n",
    "    print(\"epoch\", epoch, \"last batch\", evaluate(batch_sen_en, batch_sen_en_length, predictions, extend_vocabulary))\n",
    "\n",
    "print(\"train\", get_total_accuracy(\n",
    "    train_sen_en, train_sen_de, train_sen_en_length, train_sen_de_length, extend_vocabulary\n",
    "))\n",
    "print(\"test\", get_total_accuracy(\n",
    "    test_sen_en, test_sen_de, test_sen_en_length, test_sen_de_length, extend_vocabulary\n",
    ")) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}