imdb_ae_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## imdb dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(train_pos) 12500\n",
      "len(train_neg) 12500\n",
      "len(test_pos) 12500\n",
      "len(test_neg) 12500\n",
      "train_pos[0] ['``', 'The', 'Last', 'Hard', 'Men', \"''\", 'is', 'a', 'typical', 'western', 'for', 'the', '70', \"'s\", '.', 'Most', 'of', 'them', 'seem', 'to', 'be', 'inspired', 'by', 'Sam', 'Peckinpah', '.', 'Also', 'this', 'one', ',', 'but', 'Director', 'Andrew', 'McLaglan', 'is', 'a', 'John', 'Ford', 'Pupil', 'and', 'this', 'can', 'be', 'obviously', 'shown', 'in', 'many', 'scenes', '.', 'IMO', 'the', 'beginning', 'is', 'very', 'good', '.', 'In', 'a', 'certain', 'way', 'McLaglan', 'wanted', 'to', 'show', 'the', 'audience', 'a', 'travel', 'from', 'the', 'civilization', 'to', 'the', 'wilderness', '.', 'In', 'the', 'third', 'part', 'there', 'are', 'some', 'illogical', 'flaws', 'and', 'I', 'complain', 'a', 'bit', 'about', 'Charlton', 'Heston', '.', 'He', 'has', 'to', 'play', 'an', 'old', 'ex-lawman', 'named', 'Sam', 'Burgade', 'but', 'he', 'is', 'in', 'a', 'fantastic', 'physical', 'shape', '.', 'I', 'never', 'got', 'the', 'feeling', 'that', 'he', 'really', 'has', 'problems', 'to', 'climb', 'on', 'a', 'horse', 'or', 'on', 'a', 'rock', '.', 'For', 'me', 'he', 'did', \"n't\", 'looks', 'very', 'motivated', 'as', 'he', 'usual', 'do', 'in', 'most', 'of', 'his', 'epic', 'movies', '.', 'Same', 'goes', 'to', 'the', 'beautiful', 'Barbara', 'Hershey', 'who', 'is', 'playing', 'the', 'sheriff', \"'s\", 'daughter', '.', 'Maybe', 'both', 'had', 'troubles', 'with', 'the', 'director', 'or', 'were', 'unhappy', 'with', 'their', 'roles', '.', 'Hershey', 'and', 'Coburn', 'are', 'not', 'showing', 'their', 'best', 'but', 'they', 'are', 'still', 'good', '.', 'If', 'the', 'scriptwriter', 'had', 'John', 'Wayne', 'in', 'their', 'mind', 'as', 'Sam', 'Burgade', '?', 'Also', 'Michael', 'Parks', 'as', 'modern', 'sheriff', 'is', 'a', 'bit', 'underused', 'in', 'his', 'role', '.', 'On', 'the', 'other', 'Hand', 'there', 'is', 'James', 'Coburn', 'as', 'outlaw', 'Zach', 'Provo', '.', 'Coburn', 'is', 'a', 'really', 'great', 'villain', 'in', 'this', 'one', '.', 'He', 'is', 'portraying', 'the', 'bad', 'guy', 'between', 'maniac', 'hate', 'and', 'cleverness', '.', 'His', 'role', 'and', 'his', 'acting', 'is', 'the', 'best', 'of', 'the', 'movie.', '<', 'br', '/', '>', '<', 'br', '/', '>', 'Landscapes', 'and', 'Shootouts', 'are', 'terrific', '.', 'The', 'shootings', 'scenes', 'are', 'bloody', 'and', 'the', 'violence', 'looks', 'realistic', '.', 'Zach', 'Provo', 'and', 'his', 'gang', 'had', 'some', 'gory', 'and', 'violent', 'scenes', '.', 'What', 'I', 'miss', 'is', 'the', 'typical', 'western', 'action', 'in', 'the', 'middle', 'of', 'the', 'movie', '.', 'I', 'would', 'have', 'appreciated', 'a', 'bank', 'robbery', 'or', 'something', 'similar', '.', 'Overall', 'it', \"'s\", 'an', 'entertaining', 'western', 'flick', '.', 'Not', 'a', 'great', 'movie', 'but', 'above', 'the', 'average', 'because', 'of', 'a', 'great', 'Coburn', ',', 'a', 'very', 'good', 'beginning', 'and', 'some', 'gory', 'and', 'violent', 'scenes', '.']\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import collections\n",
    "import re\n",
    "import os\n",
    "import nltk\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def get_sent_in_directory(path_directory):\n",
    "    sents = []\n",
    "    for filename in os.listdir(path_directory):\n",
    "        if os.path.isfile(os.path.join(path_directory, filename)):\n",
    "            with open(os.path.join(path_directory, filename), 'r') as f:\n",
    "                for line in f:\n",
    "                    sents.append(nltk.tokenize.word_tokenize(line))\n",
    "    return sents\n",
    "                    \n",
    "                \n",
    "train_pos = get_sent_in_directory(\"./aclImdb/train/pos\")\n",
    "train_neg = get_sent_in_directory(\"./aclImdb/train/neg\")\n",
    "test_pos = get_sent_in_directory(\"./aclImdb/test/pos\")\n",
    "test_neg = get_sent_in_directory(\"./aclImdb/test/pos\")\n",
    "\n",
    "print(\"len(train_pos)\", len(train_pos))\n",
    "print(\"len(train_neg)\", len(train_neg))\n",
    "print(\"len(test_pos)\", len(test_pos))\n",
    "print(\"len(test_neg)\", len(test_neg))\n",
    "print(\"train_pos[0]\", train_pos[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique word: 112523\n",
      "# of sents: 25000\n",
      "max len(sents[i]): 2502\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGY5JREFUeJzt3Xt0XNV59/HvA76Bb8hIvtvIUNNi\n+iYmUR0ohJCSgHHWwqS8JXab4LBonItJ0iZ9WxL+gMBiFVZf6CoJL+CAA6YBQlvyor44IUBMHbcY\nbBJuNhfLxsbyVfhe5IskP+8fzxEajGSNpJk5Mzq/z1qz5sw+Z+bsjfB+ztl7n73N3RERkew5Lu0M\niIhIOhQAREQySgFARCSjFABERDJKAUBEJKMUAEREMkoBQEQkoxQAREQySgFARCSjBqSdgWOprq72\n2tratLMhIlJRXnzxxXfdvaa748o6ANTW1rJq1aq0syEiUlHMbGM+x6kJSEQkoxQAREQySgFARCSj\nFABERDJKAUBEJKO6DQBmNsnMlprZGjNbbWbfTtJvMLPNZvZS8pqV853vmVmDmb1pZhfnpM9M0hrM\n7NriFElERPKRzzDQVuC77v5bMxsOvGhmTyX7/tHd/3fuwWY2DZgDnAmMB542s9OT3XcCnwUagZVm\nVu/uawpREBER6ZluA4C7bwW2Jtv7zex1YMIxvjIbeMTdDwFvm1kDMCPZ1+Du6wHM7JHkWAUAEZFc\nixfDwYMwf35RT9OjPgAzqwXOAp5Pkq4xs1fMbJGZVSVpE4BNOV9rTNK6Sj/6HPPNbJWZrWpqaupJ\n9kRE+oef/hR+8pOinybvAGBmw4B/A/7K3fcBdwGnAdOJO4TbCpEhd1/o7nXuXldT0+2TzCIi0kt5\nTQVhZgOJyv+n7v4YgLtvz9n/Y+D/JR83A5Nyvj4xSeMY6SIiUmL5jAIy4D7gdXe/PSd9XM5hnwde\nS7brgTlmNtjMpgBTgReAlcBUM5tiZoOIjuL6whRDRER6Kp87gHOBLwGvmtlLSdr3gblmNh1wYAPw\nVQB3X21mjxKdu63AAndvAzCza4AngeOBRe6+uoBlERGRHshnFNBywDrZteQY37kZuLmT9CXH+p6I\niJSOngQWEckoBQARkYxSABARySgFABGRjFIAEBHJKAUAEZGMUgAQEckoBQARkYxSABARySgFABGR\njFIAEBEpN+4lOY0CgIhIObLOpmArLAUAEZGMUgAQEckoBQARkYxSABARySgFABGRjFIAEBHJKAUA\nEZGMUgAQEckoBQARkYxSABARySgFABGRjFIAEBHJKAUAEZGMUgAQEckoBQARkYxSABARKTdaEEZE\nJMO0IIyIiBRLtwHAzCaZ2VIzW2Nmq83s20n6KDN7yszWJu9VSbqZ2R1m1mBmr5jZx3J+a15y/Foz\nm1e8YomISHfyuQNoBb7r7tOAs4EFZjYNuBZ4xt2nAs8knwEuAaYmr/nAXRABA7ge+AQwA7i+PWiI\niEjpdRsA3H2ru/822d4PvA5MAGYDDySHPQBclmzPBhZ7WAGcZGbjgIuBp9x9l7vvBp4CZha0NCIi\nkrce9QGYWS1wFvA8MMbdtya7tgFjku0JwKacrzUmaV2li4hICvIOAGY2DPg34K/cfV/uPnd3oCDj\nlsxsvpmtMrNVTU1NhfhJERHpRF4BwMwGEpX/T939sSR5e9K0Q/K+I0nfDEzK+frEJK2r9A9w94Xu\nXufudTU1NT0pi4iI9EA+o4AMuA943d1vz9lVD7SP5JkHPJ6TfmUyGuhsYG/SVPQkcJGZVSWdvxcl\naSIikoIBeRxzLvAl4FUzeylJ+z5wC/ComV0NbASuSPYtAWYBDUAzcBWAu+8ys5uAlclxN7r7roKU\nQkREeqzbAODuy4GuHkm7sJPjHVjQxW8tAhb1JIMiIlIcehJYRCSjFABERDJKAUBEJKMUAEREMkoB\nQEQkoxQARETKjRaEERHJMC0IIyKSQboDEBHJKHfdAYiIZJICgIhIhikAiIhkkPoAREQySk1AIiIZ\npQAgIpJhCgAiIhmkPgARkYxSE5CISIYpAIiIZJCagEREMkpNQCIiGaUAICKSYQoAIiIZpD4AEZGM\nUhOQiEhGKQCIiGSYAoCISAapD0BEJKPUBCQikmEKACIiGaQmIBGRjCqXJiAzW2RmO8zstZy0G8xs\ns5m9lLxm5ez7npk1mNmbZnZxTvrMJK3BzK4tfFFERPqJcgkAwP3AzE7S/9HdpyevJQBmNg2YA5yZ\nfOf/mNnxZnY8cCdwCTANmJscKyIiRytRABjQfT58mZnV5vl7s4FH3P0Q8LaZNQAzkn0N7r4ewMwe\nSY5d0+Mci4j0d+5wXPFb6PtyhmvM7JWkiagqSZsAbMo5pjFJ6ypdRESOduRI2TQBdeYu4DRgOrAV\nuK1QGTKz+Wa2ysxWNTU1FepnRUQqRxn1AXyIu2939zZ3PwL8mI5mns3ApJxDJyZpXaV39tsL3b3O\n3etqamp6kz0RkcpWzk1AZjYu5+PngfYRQvXAHDMbbGZTgKnAC8BKYKqZTTGzQURHcX3vsy0i0o+V\nqAmo205gM3sYuACoNrNG4HrgAjObDjiwAfgqgLuvNrNHic7dVmCBu7clv3MN8CRwPLDI3VcXvDQi\nIv1Bie4A8hkFNLeT5PuOcfzNwM2dpC8BlvQodyIiWVTmncAiIlIs5dwJLCIiRVTOncAiIlJEagIS\nEcko3QGIiGSU7gBERDJKncAiIhmlJiARkYxSE5CISEbpDkBEJKN0ByAiklG6AxARySjdAYiIZJSG\ngYqIZJSagEREMkpNQCIiGaU7ABGRjNIdgIhIRqkTWEQko9QEJCKSUWoCEhHJKN0BiIhklO4AREQy\nSp3AIiIZdeSImoBERDJJdwAiIhmlTmARkYzSHYCISAYdPhzvAwcW/VQKACIi5WT37nivqir6qRQA\nRETKSXsAGDWq6KdSABARKSe7dsV7OdwBmNkiM9thZq/lpI0ys6fMbG3yXpWkm5ndYWYNZvaKmX0s\n5zvzkuPXmtm84hRHRKTCldkdwP3AzKPSrgWecfepwDPJZ4BLgKnJaz5wF0TAAK4HPgHMAK5vDxoi\nIpLj5Zfjfdy4op+q2wDg7suAXUclzwYeSLYfAC7LSV/sYQVwkpmNAy4GnnL3Xe6+G3iKDwcVEZFs\na2uDH/0IzjsPJk0q+ul62wcwxt23JtvbgDHJ9gRgU85xjUlaV+kfYmbzzWyVma1qamrqZfZERCrQ\nkiWwdSt8+9slOV2fO4Hd3QEvQF7af2+hu9e5e11NTU2hflZEpPzdfTdUV8Oll5bkdL0NANuTph2S\n9x1J+mYg975lYpLWVbqIiEA8APb00zBnDgwaVJJT9jYA1APtI3nmAY/npF+ZjAY6G9ibNBU9CVxk\nZlVJ5+9FSZqIiEB0/h4+DJ/8ZMlOOaC7A8zsYeACoNrMGonRPLcAj5rZ1cBG4Irk8CXALKABaAau\nAnD3XWZ2E7AyOe5Gdz+6Y1lEJLuWL4/3c88t2Sm7DQDuPreLXRd2cqwDC7r4nUXAoh7lTkQkK154\nASZOhAmdjo8pCj0JLCKSNnd49tkY/llCCgAiImlbswa2bYPPfKakp1UAEBFJ23/9V7x/6lMlPa0C\ngIhI2p54Ak48EU49taSnVQAQEUmTOzz/PMyYUZJlIHMpAIiIpOm556L9/y/+ouSnVgAQEUnTgw/G\n+2WXHfu4IlAAEBFJizv8+7/D5z8fcwCVmAKAiEhali+HzZvhc59L5fQKACIiafnRj2Lpxy98IZXT\nKwCIiKShqQkefzw6f4cNSyULCgAiImm44w44dAi+/vXUsqAAICJSai0tsHAhzJoF06allg0FABGR\nUnvoIdixI9Wrf1AAEBEprdZWuO46mD497gBSpAAgIlJKP/95DP38zndKPvXD0RQARERK5cABuPZa\nGDMGrrii++OLrNsVwUREpEB+8ANYvz4Wfx88OO3c6A5ARKQk3ngDbr0V5s2DCz+0om4qFABERIrt\nwAG48ko44QT4+79POzfvUxOQiEix3X47rFwJ//zPMG5c2rl5n+4ARESK6a234MYbY8hnCnP+H4sC\ngIhIsbS0wBe/CEOGwD33pJ2bD1ETkIhIMbS0wF/+ZTT9PPQQTJyYdo4+RHcAIiKF5g7f+AYsXgx/\n+7cwd27aOeqUAoCISCG5w/e+B/feGw993Xpr2jnqkpqAREQK5fBhWLAgKv958+Dmm9PO0THpDkBE\npBBaW6Op5957Y56f++5Lfa6f7pR37kREKsHhw3HF/9hjcMstcNttcPzxaeeqW2oCEhHpiy1bYk3f\n5ctjrp+/+7u0c5S3PgUAM9sA7AfagFZ3rzOzUcDPgFpgA3CFu+82MwP+CZgFNANfdvff9uX8IiKp\neuUVmD07pndetAiuuirtHPVIIZqAPu3u0929Lvl8LfCMu08Fnkk+A1wCTE1e84G7CnBuEZHSc485\nfc46C/btg6VLK67yh+L0AcwGHki2HwAuy0lf7GEFcJKZlc+kGCIi+WhshE9/Gr7//ZjeYfVqOPfc\ntHPVK30NAA78ysxeNLP5SdoYd9+abG8DxiTbE4BNOd9tTNJERMpfWxvceWcs4v7cc9HRW18PY8em\nnbNe62sn8HnuvtnMRgNPmdkbuTvd3c3Me/KDSSCZDzB58uQ+Zk9EpAB+85sY3//qq/Dxj8ODD8IZ\nZ6Sdqz7r0x2Au29O3ncAPwdmANvbm3aS9x3J4ZuBSTlfn5ikHf2bC929zt3rampq+pI9EZG+eeml\naO45/3zYuRMWLoTnn+8XlT/0IQCY2VAzG96+DVwEvAbUA/OSw+YBjyfb9cCVFs4G9uY0FYmIlI/V\nq2Pq5rPOghdegBtugDffhK98pSLG9+erL01AY4Cfx+hOBgAPufsvzWwl8KiZXQ1sBNpXPl5CDAFt\nIIaBVl6XuYj0b6tXx/QNDz8cUzgvWADXXVdWi7gUUq8DgLuvBz7aSfpO4EMLXrq7Awt6ez4RkaJo\na4MnnoAf/jAWax84EL71raj4R49OO3dFpSeBRSSbtmyB+++Hu++GTZviKv8HP4Cvfa3fV/ztFABE\nJDva2uA//iNW53rssZjA7YILYkjn7NkwaFDaOSwpBQAR6f+2bo2r/XvugY0bYeTIaN//2tfgD/4g\n7dylRgFARPqnlpYYv794cSzJ2NIC550HN90Ef/qnMHRo2jlMnQKAiPQP7vDyy/DrX0czz9KlsH9/\njOaZMwf+5m/gIx9JO5dlRQFARCrXvn0xDfMvfgGPPx6duQCTJsHll8PMmTFfz/Dh6eazTCkAiEjl\naGuDF1+EX/4SliyJh7Tco/P2T/4khm5efDGccgrEM0pyDAoAIlK+WlqiWec3v4kr/aVLYffu2Dd9\neiy+cv758MlPwrBh6ea1AikAiEj52LsXVqyI2TaXLYt5d5qbY9+4cdGc89nPwkUX9dunc0tJAUBE\n0tHSAq+9BitXRlPO8uUx3067M8+EL30J/viP4wq/tlbNOgWmACAixdfcHDNr/u538Xr11fh8+HDs\nHzo0FlW54go45xz4xCdg1Kh085wBCgAiUlgHDsDrr3dU+CtXRsdta2vsHz48hmN+9aswY0bMuDlt\nmq7uU6AAICK9s3dvNNm8/no05bzxRsym+fbbHccMGRKV/Te/GVf2dXUxQue4YqxGKz2lACAiXTt4\nENati9ebb8I770BDA6xZE9vtjjsOpk6NkTl//udxRf+Rj8Q0CwNUzZQr/WVEsm7nzrh637gR1q6N\nin3DBli/PraPHOk4duhQmDIF/uiP4Oqro6I//fRYIWvgwNSKIL2jACDSn7nDnj1Rob/9drw2bIir\n+PXrobGxY5hlu5qaaKb5+MdjCoXf/334vd+Ldy3T2q8oAIhUKveY6+add2Ju+02bYnvz5tjeuDFe\nBw9+8HtDhkSFfsYZMa5+7Nio3Gtr4bTTNG1ChigAiJSjXbviidfGRti+PV6bNsW0xo2Nsb1jRwSA\no1VVRWV+xhnx0NTEidFsU1sbr5oajbgRQAFApDTa2uLKfM+euFrfuTPed+2Cbduign/33Y59R1+1\nQyxGPmYMTJ4MH/1oXLlPmgTjx8OECVG5T5gQV/gieVAAEOmJlpYY575tW8xE+e67UWG3v+/eHVfm\n770XV+u7d0elfuhQ1785fnxctU+cGE0z48fHkoQ1NVHJ19REpV9VpY5WKSgFAMmevXujEm+vwA8e\nhKamjvTt26Nppb2S37079u/bF1fsxzJ0aMxRM2wYVFdH2/qYMVF5V1fHa8yYeB8/HkaMiCt7kRQo\nAEhlcI8Keu/emD7gv/87KuVDh6KC3r07tvfvj/T2Sn3fvtjesSO+09zcefNKruOOiyUDR46MynrY\nsBjjPmxYXJGPGPHBfePHR8fpuHFqW5eKogAghXXkSFTQLS1RKe/fH58PHIhK+PDheO3ZE69Dh6JS\n37Ur0pub47hDhzoq7gMHjt2EcrSTT44r8aFDo5KuqYFTT433IUNijpmqqtgeO/aDx55wQlTwamqR\nDFAAyIqWlqhcW1qion333Y7Pra1RCe/b1/G5/Qq6vTJvbY027vfei88tLXH8nj0dn3fujMq6J8yi\nIh49OirfwYOjAp8wIRb5GDEiKuzBg+HEE+O4wYNj3+jRcQV+wglRkQ8ZEul68lQkL/qXUo7ee6+j\nSaOpKa6i26+Gm5ujUn7vvajEDx+O49qvlNsr7PZmkPbP+/dHM0pPDRkSV8MDB0ZFW13d8fnEE+OB\nofYKub0iHzSo4/iamo79o0ZFU0n7sWPHqrIWSZH+9ZVCa2s8kLN5c4zhbmyMCnnLlrjy3rIlrqa3\nbImKvX2K3O4MHx5NF4MGxXZVVVS2w4bF8MCqqg9WxtXVHZ+HDu2ozAcMiAo5d//AgXDSSaqgRfox\n/esulIMHYx6VhoaYNGvtWnjrrXjsvrGx8++cfHJcIdfURCfjOefEVXJ7Z+OQIVFRjx4d28OHf7CS\n1nhvEekDBYCeaG6OK/m33uqo5Net65g0K7eJpaoqJsk6//x4QGfy5Bjnfcop0b49cqSmxBWRVCkA\nHK21NSr2NWvian7dupgpce3aGBeeq6oqHtyZMQPmzo1H7087Ld61mpGIlLnsBQD3aG9fty4q+Par\n+I0bO2ZLzDViRMxpfuGFMZTw1FM7Zkasrta4bxGpWP0/AKxbB08/HcvTrVkTKxbt3PnBY0aNioq9\nrg6+8IWo4M88Myr+kSNVyYtIv1TyAGBmM4F/Ao4H7nX3W4pyom3bYN48+NWv4vPQobF4xec+FxX7\nlClR0Z9+elzli4hkTEkDgJkdD9wJfBZoBFaaWb27ryn4yebNg2XL4IYb4M/+LJpsNOeKiMj7Sn0H\nMANocPf1AGb2CDAbKGwAOHAAnn0WvvUtuP76gv60iEh/UepxiBOATTmfG5O0wtq7Fy6/HC65pOA/\nLSLSX5RdJ7CZzQfmA0yePLl3PzJ2LDz0UAFzJSLS/5T6DmAzMCnn88Qk7X3uvtDd69y9rkYLUIuI\nFE2pA8BKYKqZTTGzQcAcoL7EeRAREUrcBOTurWZ2DfAkMQx0kbuvLmUeREQklLwPwN2XAEtKfV4R\nEfkgzUYmIpJRCgAiIhmlACAiklEKACIiGWXem3ViS8TMmoCNffiJauDdAmWnUmStzFkrL6jMWdGX\nMp/i7t0+SFXWAaCvzGyVu9elnY9SylqZs1ZeUJmzohRlVhOQiEhGKQCIiGRUfw8AC9POQAqyVuas\nlRdU5qwoepn7dR+AiIh0rb/fAYiISBcqPgCY2Uwze9PMGszs2k72DzaznyX7nzez2tLnsrDyKPN3\nzGyNmb1iZs+Y2Slp5LOQuitzznGXm5mbWcWPGMmnzGZ2RfK3Xm1mFb8IRh7/b082s6Vm9rvk/+9Z\naeSzUMxskZntMLPXuthvZnZH8t/jFTP7WEEz4O4V+yJmFF0HnAoMAl4Gph11zDeAu5PtOcDP0s53\nCcr8aeDEZPvrWShzctxwYBmwAqhLO98l+DtPBX4HVCWfR6ed7xKUeSHw9WR7GrAh7Xz3scznAx8D\nXuti/yzgF4ABZwPPF/L8lX4H8P4aw+5+GGhfYzjXbOCBZPtfgQvNzEqYx0LrtszuvtTdm5OPK4iF\ndypZPn9ngJuAW4GDpcxckeRT5q8Ad7r7bgB331HiPBZaPmV2YESyPRLYUsL8FZy7LwN2HeOQ2cBi\nDyuAk8xsXKHOX+kBIJ81ht8/xt1bgb3AySXJXXH0dF3lq4kriErWbZmTW+NJ7v5EKTNWRPn8nU8H\nTjez/zSzFWY2s2S5K458ynwD8EUzaySmlf9mabKWmqKuo152awJL4ZjZF4E64FNp56WYzOw44Hbg\nyylnpdQGEM1AFxB3ecvM7H+4+55Uc1Vcc4H73f02MzsHeNDM/tDdj6SdsUpU6XcA3a4xnHuMmQ0g\nbht3liR3xZFPmTGzzwDXAZe6+6ES5a1YuivzcOAPgWfNbAPRVlpf4R3B+fydG4F6d29x97eBt4iA\nUKnyKfPVwKMA7v4cMISYM6e/yuvfe29VegDIZ43hemBesv0/gV970rtSobots5mdBdxDVP6V3i4M\n3ZTZ3fe6e7W717p7LdHvcam7r0onuwWRz//b/5e4+sfMqokmofWlzGSB5VPmd4ALAczsDCIANJU0\nl6VVD1yZjAY6G9jr7lsL9eMV3QTkXawxbGY3AqvcvR64j7hNbCA6W+akl+O+y7PM/wAMA/4l6e9+\nx90vTS3TfZRnmfuVPMv8JHCRma0B2oD/5e4Ve3ebZ5m/C/zYzP6a6BD+ciVf0JnZw0QQr076Na4H\nBgK4+91EP8csoAFoBq4q6Pkr+L+diIj0QaU3AYmISC8pAIiIZJQCgIhIRikAiIhklAKAiEhGKQCI\niGSUAoCISEYpAIiIZNT/BxT2MyCQ1jP1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbfd9af0dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "corpus_counter = collections.Counter()\n",
    "train_preprocess = []\n",
    "for sent in train_pos:\n",
    "    res = []\n",
    "    for word in sent:\n",
    "        if re.search('[a-zA-Z]', word):\n",
    "            corpus_counter[word.lower()] += 1\n",
    "            res.append(word.lower())\n",
    "    train_preprocess.append((res, 1))\n",
    "for sent in train_neg:\n",
    "    res = []\n",
    "    for word in sent:\n",
    "        if re.search('[a-zA-Z]', word):\n",
    "            corpus_counter[word.lower()] += 1\n",
    "            res.append(word.lower())\n",
    "    train_preprocess.append((res, 0))\n",
    "\n",
    "print(\"unique word:\", len(corpus_counter))\n",
    "print(\"# of sents:\", len(train_preprocess))\n",
    "print(\"max len(sents[i]):\", max([len(s) for s, _ in train_preprocess]))\n",
    "\n",
    "train_length = []\n",
    "for sent, label in train_preprocess:\n",
    "    train_length.append(len(sent))\n",
    "train_length = sorted(train_length)\n",
    "stat_x, stat_y = [], []\n",
    "for i, l in enumerate(train_length):\n",
    "    stat_x.append(i / len(train_length))\n",
    "    stat_y.append(l)\n",
    "plt.plot(stat_x, stat_y, 'r-')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use Brown Corpus to train an seq2seq autoencoder "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import random\n",
    "\n",
    "random.seed(1337)\n",
    "\n",
    "class EncoderDecoder:\n",
    "    def __init__(self, vocabulary={}, state_size=64, n_max_length=30):     \n",
    "        self.state_size = state_size\n",
    "        self.n_max_length = n_max_length\n",
    "        self.vocabulary = vocabulary\n",
    "        self.reverse_vocabulary = {k: v for k, v in vocabulary.items()}\n",
    "        \n",
    "        ######################\n",
    "        # Graph Construction #\n",
    "        ######################\n",
    "        self.graph = tf.Graph()\n",
    "        with self.graph.as_default():\n",
    "            #self.sen_en = tf.placeholder(tf.int32, shape=(None, self.n_max_length), name=\"sen_en\")\n",
    "            #self.sen_de = tf.placeholder(tf.int32, shape=(None, self.n_max_length), name=\"sen_de\")\n",
    "            self.sen_en = tf.placeholder(tf.int32, shape=(None, None), name=\"sen_en\")\n",
    "            self.sen_de = tf.placeholder(tf.int32, shape=(None, None), name=\"sen_de\")\n",
    "            self.sen_en_length = tf.placeholder(tf.int32, shape=(None,), name=\"sen_en_length\")\n",
    "            self.sen_de_length = tf.placeholder(tf.int32, shape=(None,), name=\"sen_de_length\")\n",
    "            \n",
    "            batch_size_en = tf.shape(self.sen_en)[0]\n",
    "            batch_size_de = tf.shape(self.sen_de)[0]\n",
    "            batch_max_length_de = tf.shape(self.sen_de)[1]\n",
    "            \n",
    "            # TODO sen_en_embedding could also be self-trained embedding: embedding_lookup\n",
    "            self.embedding = tf.Variable(tf.random_uniform([len(self.vocabulary), self.state_size], -1.0, 1.0), dtype=tf.float32)\n",
    "            #self.sen_en_embedding = tf.one_hot(self.sen_en, len(self.vocabulary))\n",
    "            #self.sen_de_embedding = tf.one_hot(self.sen_de, len(self.vocabulary))\n",
    "            self.sen_en_embedding = tf.nn.embedding_lookup(self.embedding, self.sen_en)\n",
    "            self.sen_de_embedding = tf.nn.embedding_lookup(self.embedding, self.sen_de)\n",
    "            \n",
    "            # build encoder decoder structure\n",
    "            with tf.variable_scope(\"encoder\") as scope:\n",
    "                self.cell_en = tf.contrib.rnn.BasicLSTMCell(self.state_size)\n",
    "            with tf.variable_scope(\"decoder\") as scope:\n",
    "                self.cell_de = tf.contrib.rnn.BasicLSTMCell(self.state_size)\n",
    "            with tf.variable_scope(\"encoder\") as scope:\n",
    "                self.cell_en_init = self.cell_en.zero_state(batch_size_en, tf.float32)\n",
    "                self.h_state_en, self.final_state_en = tf.nn.dynamic_rnn(\n",
    "                    self.cell_en,\n",
    "                    self.sen_en_embedding,\n",
    "                    sequence_length=self.sen_en_length,\n",
    "                    initial_state=self.cell_en_init,\n",
    "                )\n",
    "            with tf.variable_scope(\"decoder\") as scope:\n",
    "                self.cell_de_init = self.final_state_en\n",
    "                self.h_state_de, self.final_state_de = tf.nn.dynamic_rnn(\n",
    "                    self.cell_de,\n",
    "                    self.sen_en_embedding,\n",
    "                    sequence_length=self.sen_en_length,\n",
    "                    initial_state=self.cell_de_init,\n",
    "                )\n",
    "            \n",
    "            with tf.variable_scope(\"softmax\") as scope:\n",
    "                W = tf.get_variable(\"W\", [self.state_size, len(self.vocabulary)], initializer=tf.random_normal_initializer(seed=None))\n",
    "                b = tf.get_variable(\"b\", [len(self.vocabulary)], initializer=tf.random_normal_initializer(seed=None))               \n",
    "            self.logits = tf.reshape(\n",
    "                tf.add(tf.matmul(tf.reshape(self.h_state_de, (-1, self.state_size)), W), b),\n",
    "                shape=(-1, batch_max_length_de, len(self.vocabulary))\n",
    "            )\n",
    "            self.prediction = tf.nn.softmax(self.logits)\n",
    "                \n",
    "            # construct loss and train op\n",
    "            self.cross_ent = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                labels=self.sen_de,\n",
    "                logits=self.logits\n",
    "            )        \n",
    "            self.mask = tf.sequence_mask(self.sen_de_length, maxlen=batch_max_length_de)\n",
    "            self.loss = tf.reduce_mean(\n",
    "                #tf.reduce_sum(tf.multiply(self.cross_ent, self.mask), 1) / tf.reduce_sum(self.mask, 1)\n",
    "                tf.divide(\n",
    "                    tf.reduce_sum(\n",
    "                        tf.where(\n",
    "                            self.mask,\n",
    "                            self.cross_ent,\n",
    "                            tf.zeros_like(self.cross_ent)\n",
    "                        ), 1\n",
    "                    ),\n",
    "                    tf.to_float(self.sen_de_length)\n",
    "                )\n",
    "            )\n",
    "            \n",
    "            \"\"\"\n",
    "            optimizer = tf.train.AdamOptimizer()\n",
    "            self.op_train = optimizer.minimize(self.loss)\n",
    "            \"\"\"\n",
    "            # Calculate and clip gradients\n",
    "            params = tf.trainable_variables()\n",
    "            gradients = tf.gradients(self.loss, params)\n",
    "            self.clipped_gradients, _ = tf.clip_by_global_norm(gradients, 1)\n",
    "            # Optimization\n",
    "            optimizer = tf.train.AdamOptimizer()\n",
    "            self.op_train = optimizer.apply_gradients(zip(self.clipped_gradients, params))\n",
    "            \n",
    "            # initializer\n",
    "            gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.2)\n",
    "            self.sess = tf.Session(\n",
    "                graph=self.graph,\n",
    "                config=tf.ConfigProto(gpu_options=gpu_options)\n",
    "            )           \n",
    "            self.init = tf.global_variables_initializer()\n",
    "            self.sess.run(self.init)\n",
    "            \n",
    "    def train(self, batch_sen_en, batch_sen_de, batch_sen_en_length, batch_sen_de_length):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        batch_sen_en: numpy, shape=(n, max_length), dtype=int\n",
    "        batch_sen_de: numpy, shape=(n, max_length), dtype=int\n",
    "        batch_sen_en_length: numpy, shape=(n,), dtype=int\n",
    "        batch_sen_de_length: numpy, shape=(n,), dtype=int\n",
    "        \"\"\"\n",
    "        assert batch_sen_en.shape[0] == batch_sen_de.shape[0]\n",
    "        assert batch_sen_en.shape[1] == self.n_max_length  # training always input same length as self.n_max_length\n",
    "        _, loss, prediction, sen_en_embedding, mask, cross_ent, clipped_gradients = self.sess.run(\n",
    "            [self.op_train, self.loss, self.prediction, self.sen_en_embedding, self.mask, self.cross_ent, self.clipped_gradients],\n",
    "            feed_dict={\n",
    "                self.sen_en: batch_sen_en,\n",
    "                self.sen_de: batch_sen_de,\n",
    "                self.sen_en_length: batch_sen_en_length,\n",
    "                self.sen_de_length: batch_sen_de_length,\n",
    "            }\n",
    "        )\n",
    "        return loss, prediction, sen_en_embedding, mask, cross_ent, clipped_gradients\n",
    "        \n",
    "    def predict(self, batch_sen_en, batch_sen_de, batch_sen_en_length, batch_sen_de_length):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        batch_sen_en: numpy, shape=(n, max_length), dtype=int\n",
    "        batch_sen_de: numpy, shape=(n, max_length), dtype=int\n",
    "        batch_sen_en_length: numpy, shape=(n,), dtype=int\n",
    "        batch_sen_de_length: numpy, shape=(n,), dtype=int\n",
    "        \"\"\"\n",
    "        assert batch_sen_en.shape[0] == batch_sen_de.shape[0]\n",
    "        loss, prediction = self.sess.run(\n",
    "            [self.loss, self.prediction],\n",
    "            feed_dict={\n",
    "                self.sen_en: batch_sen_en,\n",
    "                self.sen_de: batch_sen_de,\n",
    "                self.sen_en_length: batch_sen_en_length,\n",
    "                self.sen_de_length: batch_sen_de_length,\n",
    "            }\n",
    "        )\n",
    "        return loss, prediction\n",
    "    \n",
    "    def encode(self, batch_sen_en, batch_sen_en_length):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        batch_sen_en: numpy, shape=(n, max_length), dtype=int\n",
    "        batch_sen_en_length: numpy, shape=(n,), dtype=int\n",
    "        Returns\n",
    "        -------\n",
    "        #batch_state_en: numpy, shape=(n, self.state_size), dtype=float\n",
    "        batch_state_en: LSTMStateTuple\n",
    "        \"\"\"\n",
    "        batch_state_en = self.sess.run(\n",
    "            self.final_state_en,\n",
    "            feed_dict={\n",
    "                self.sen_en: batch_sen_en,\n",
    "                self.sen_en_length: batch_sen_en_length,\n",
    "            }\n",
    "        )\n",
    "        return batch_state_en\n",
    "    \n",
    "    def decode(self, batch_state_en):\n",
    "        \"\"\"\n",
    "        Parameters\n",
    "        ----------\n",
    "        batch_state_en: LSTMStateTuple\n",
    "        Returns\n",
    "        -------\n",
    "        batch_sen_de: numpy, shape=(n, max_length), dtype=int\n",
    "        \"\"\"\n",
    "        batch_size = batch_state_en.c.shape[0]\n",
    "        batch_sen_de = np.empty([batch_size, self.n_max_length], dtype=np.int32)\n",
    "        \n",
    "        tmp_sen_de = np.empty([batch_size, 1], dtype=np.int32)\n",
    "        tmp_sen_de_length = np.ones([batch_size], dtype=np.int32)\n",
    "        tmp_sen_de[:] = self.vocabulary[\"<s>\"]\n",
    "        tmp_last_state = batch_state_en\n",
    "        for i in range(self.n_max_length):\n",
    "            tmp_predict, tmp_last_state = self.sess.run(\n",
    "                [self.prediction, self.final_state_de],\n",
    "                feed_dict={\n",
    "                    self.cell_de_init: tmp_last_state,\n",
    "                    self.sen_de: tmp_sen_de,\n",
    "                    self.sen_de_length: tmp_sen_de_length,\n",
    "                }\n",
    "            )\n",
    "            tmp_sen_de = np.argmax(tmp_predict, axis=2)\n",
    "            batch_sen_de[:,i] = tmp_sen_de[:,0]\n",
    "           \n",
    "        return batch_sen_de\n",
    "\n",
    "    \n",
    "def evaluate(batch_sen_en, batch_sen_en_length, batch_prediction, vocabulary):\n",
    "    \"\"\"\n",
    "    Parameters\n",
    "    ----------\n",
    "    batch_sen_en: numpy, shape=(n, max_length), dtype=int\n",
    "    batch_sen_en_length: numpy, shape=(n,), dtype=int\n",
    "    batch_prediction: numpy, shape=(n, max_length, len(vocabulary))\n",
    "    \"\"\"\n",
    "    assert batch_sen_en.shape[0] == batch_prediction.shape[0]\n",
    "    acc_word = 0\n",
    "    acc_sen_end = 0\n",
    "    for i in range(batch_sen_en.shape[0]):\n",
    "        is_first_end = False\n",
    "        for j in range(batch_sen_en_length[i]):\n",
    "            cur_pred_word = np.argmax(batch_prediction[i, j])\n",
    "            if cur_pred_word == batch_sen_en[i, j]:\n",
    "                acc_word += 1\n",
    "                if not is_first_end and cur_pred_word == vocabulary[\"</s>\"]:\n",
    "                    acc_sen_end += 1\n",
    "            if cur_pred_word == vocabulary[\"</s>\"]:\n",
    "                is_first_end = True\n",
    "    return 1. * acc_word / np.sum(batch_sen_en_length), 1. * acc_sen_end / batch_sen_en.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n_train 20000 n_valid 5000\n",
      "[20002     0   224   259   331     5     2   745   985    15     0    12\n",
      "    90     3    97   290     4    28  1580    33  1183  8174    82    10\n",
      "    29    18   161  3048 20001     5     2   288  1660 15465     1    10\n",
      "    60    28   503   576     8   107   138  6320     0   436     5    54\n",
      "    50     8     2   750    95 20001   447     4   118     0   300     2\n",
      "  2060    36     0  4549     4     0  5329     8     0   815   169    39\n",
      "    24    49  4182  1487     1     9  3735     2   217    44  5714  3225\n",
      "    25    45     4   293    34   164 20001   728  1183 14110    18    25\n",
      "     5     8     2 20003]\n",
      "['the', 'last', 'hard', 'men', 'is', 'a', 'typical', 'western', 'for', 'the', \"'s\", 'most', 'of', 'them', 'seem', 'to', 'be', 'inspired', 'by', 'sam', 'peckinpah', 'also', 'this', 'one', 'but', 'director', 'andrew', 'mclaglan', 'is', 'a', 'john', 'ford', 'pupil', 'and', 'this', 'can', 'be', 'obviously', 'shown', 'in', 'many', 'scenes', 'imo', 'the', 'beginning', 'is', 'very', 'good', 'in', 'a', 'certain', 'way', 'mclaglan', 'wanted', 'to', 'show', 'the', 'audience', 'a', 'travel', 'from', 'the', 'civilization', 'to', 'the', 'wilderness', 'in', 'the', 'third', 'part', 'there', 'are', 'some', 'illogical', 'flaws', 'and', 'i', 'complain', 'a', 'bit', 'about', 'charlton', 'heston', 'he', 'has', 'to', 'play', 'an', 'old', 'ex-lawman', 'named', 'sam', 'burgade', 'but', 'he', 'is', 'in', 'a', 'fantastic', 'physical', 'shape', 'i', 'never', 'got', 'the', 'feeling', 'that', 'he', 'really', 'has', 'problems', 'to', 'climb', 'on', 'a', 'horse', 'or', 'on', 'a', 'rock', 'for', 'me', 'he', 'did', \"n't\", 'looks', 'very', 'motivated', 'as', 'he', 'usual', 'do', 'in', 'most', 'of', 'his', 'epic', 'movies', 'same', 'goes', 'to', 'the', 'beautiful', 'barbara', 'hershey', 'who', 'is', 'playing', 'the', 'sheriff', \"'s\", 'daughter', 'maybe', 'both', 'had', 'troubles', 'with', 'the', 'director', 'or', 'were', 'unhappy', 'with', 'their', 'roles', 'hershey', 'and', 'coburn', 'are', 'not', 'showing', 'their', 'best', 'but', 'they', 'are', 'still', 'good', 'if', 'the', 'scriptwriter', 'had', 'john', 'wayne', 'in', 'their', 'mind', 'as', 'sam', 'burgade', 'also', 'michael', 'parks', 'as', 'modern', 'sheriff', 'is', 'a', 'bit', 'underused', 'in', 'his', 'role', 'on', 'the', 'other', 'hand', 'there', 'is', 'james', 'coburn', 'as', 'outlaw', 'zach', 'provo', 'coburn', 'is', 'a', 'really', 'great', 'villain', 'in', 'this', 'one', 'he', 'is', 'portraying', 'the', 'bad', 'guy', 'between', 'maniac', 'hate', 'and', 'cleverness', 'his', 'role', 'and', 'his', 'acting', 'is', 'the', 'best', 'of', 'the', 'movie.', 'br', 'br', 'landscapes', 'and', 'shootouts', 'are', 'terrific', 'the', 'shootings', 'scenes', 'are', 'bloody', 'and', 'the', 'violence', 'looks', 'realistic', 'zach', 'provo', 'and', 'his', 'gang', 'had', 'some', 'gory', 'and', 'violent', 'scenes', 'what', 'i', 'miss', 'is', 'the', 'typical', 'western', 'action', 'in', 'the', 'middle', 'of', 'the', 'movie', 'i', 'would', 'have', 'appreciated', 'a', 'bank', 'robbery', 'or', 'something', 'similar', 'overall', 'it', \"'s\", 'an', 'entertaining', 'western', 'flick', 'not', 'a', 'great', 'movie', 'but', 'above', 'the', 'average', 'because', 'of', 'a', 'great', 'coburn', 'a', 'very', 'good', 'beginning', 'and', 'some', 'gory', 'and', 'violent', 'scenes']\n",
      "['<s>', 'the', 'last', 'hard', 'men', 'is', 'a', 'typical', 'western', 'for', 'the', \"'s\", 'most', 'of', 'them', 'seem', 'to', 'be', 'inspired', 'by', 'sam', 'peckinpah', 'also', 'this', 'one', 'but', 'director', 'andrew', '<unk>', 'is', 'a', 'john', 'ford', 'pupil', 'and', 'this', 'can', 'be', 'obviously', 'shown', 'in', 'many', 'scenes', 'imo', 'the', 'beginning', 'is', 'very', 'good', 'in', 'a', 'certain', 'way', '<unk>', 'wanted', 'to', 'show', 'the', 'audience', 'a', 'travel', 'from', 'the', 'civilization', 'to', 'the', 'wilderness', 'in', 'the', 'third', 'part', 'there', 'are', 'some', 'illogical', 'flaws', 'and', 'i', 'complain', 'a', 'bit', 'about', 'charlton', 'heston', 'he', 'has', 'to', 'play', 'an', 'old', '<unk>', 'named', 'sam', 'burgade', 'but', 'he', 'is', 'in', 'a', '</s>']\n",
      "[20002   219  1219   182   198  6100     9    80   111    79   144   108\n",
      "   592     3    57   119   142     0   219   771    21  5603  8433     1\n",
      "  4080    21    26  2691   306    48    13     9   505    89    72    10\n",
      "  1125     2  1763  1719     9  2684    25   429    26   311    18    84\n",
      "    74    11     0    17    13   321  4085     4    71    48    13     0\n",
      "  1239     3  1219    37    75     1   165     4     0  4353     3     2\n",
      "  2988    15    58  1641   272    89    72    10 20001    26   119   134\n",
      "    72    70    27     4    68     0  1636   164   700    21    40  6577\n",
      "   134     6     6 20003]\n",
      "['guy', 'walking', 'around', 'without', 'motive', 'i', 'will', 'never', 'get', 'those', 'two', 'hours', 'of', 'my', 'life', 'back', 'the', 'guy', 'kept', 'on', 'assuming', 'identities', 'and', 'cheating', 'on', 'his', 'pregnant', 'wife', 'what', 'was', 'i', 'thinking', 'how', 'did', 'this', 'win', 'a', 'price', 'anywhere', 'i', 'understood', 'he', 'loved', 'his', 'father', 'but', 'other', 'than', 'that', 'the', 'movie', 'was', 'completely', 'senseless', 'to', 'me', 'what', 'was', 'the', 'purpose', 'of', 'walking', 'so', 'much', 'and', 'going', 'to', 'the', 'funeral', 'of', 'a', 'stranger', 'for', 'no', 'apparent', 'reason', 'how', 'did', 'this', 'enrich', 'his', 'life', 'why', 'did', 'we', 'have', 'to', 'see', 'the', 'dying', 'old', 'lady', 'on', 'her', 'underwear', 'why', 'br', 'br', 'i', 'though', 'it', 'would', 'be', 'deep', 'or', 'about', 'something', 'more', 'interesting', 'i', 'do', 'not', 'recommend', 'the', 'movie', 'even', 'to', 'leave', 'on', 'while', 'sleeping']\n",
      "['<s>', 'guy', 'walking', 'around', 'without', 'motive', 'i', 'will', 'never', 'get', 'those', 'two', 'hours', 'of', 'my', 'life', 'back', 'the', 'guy', 'kept', 'on', 'assuming', 'identities', 'and', 'cheating', 'on', 'his', 'pregnant', 'wife', 'what', 'was', 'i', 'thinking', 'how', 'did', 'this', 'win', 'a', 'price', 'anywhere', 'i', 'understood', 'he', 'loved', 'his', 'father', 'but', 'other', 'than', 'that', 'the', 'movie', 'was', 'completely', 'senseless', 'to', 'me', 'what', 'was', 'the', 'purpose', 'of', 'walking', 'so', 'much', 'and', 'going', 'to', 'the', 'funeral', 'of', 'a', 'stranger', 'for', 'no', 'apparent', 'reason', 'how', 'did', 'this', '<unk>', 'his', 'life', 'why', 'did', 'we', 'have', 'to', 'see', 'the', 'dying', 'old', 'lady', 'on', 'her', 'underwear', 'why', 'br', 'br', '</s>']\n",
      "epoch 0 valid (0.12444843916294056, 0.0)\n",
      "last loss 5.96657\n",
      "epoch 1 valid (0.14404928636808528, 0.0008)\n",
      "last loss 5.70911\n",
      "epoch 2 valid (0.15195246283204245, 0.0004)\n",
      "last loss 5.56068\n",
      "epoch 3 valid (0.15994304726972963, 0.0004)\n",
      "last loss 5.44594\n",
      "epoch 4 valid (0.16337115850320799, 0.0024)\n",
      "last loss 5.36575\n",
      "epoch 5 valid (0.16809805267395167, 0.0144)\n",
      "last loss 5.28484\n",
      "epoch 6 valid (0.17169064749561297, 0.08)\n",
      "last loss 5.21827\n",
      "epoch 7 valid (0.17641957719785892, 0.1888)\n",
      "last loss 5.16194\n",
      "epoch 8 valid (0.17717100436162989, 0.2124)\n",
      "last loss 5.11151\n",
      "epoch 9 valid (0.18090719221740675, 0.3344)\n",
      "last loss 5.07797\n",
      "epoch 10 valid (0.18582146948063774, 0.2892)\n",
      "last loss 5.03934\n",
      "epoch 11 valid (0.18532608088421745, 0.3702)\n",
      "last loss 5.00968\n",
      "epoch 12 valid (0.1894285217269083, 0.1396)\n",
      "last loss 4.98404\n",
      "epoch 13 valid (0.19147466908114544, 0.2018)\n",
      "last loss 4.9491\n",
      "epoch 14 valid (0.19189926577468941, 0.1932)\n",
      "last loss 4.94144\n",
      "epoch 15 valid (0.19230091717161074, 0.0568)\n",
      "last loss 4.91275\n",
      "epoch 16 valid (0.1933583398979207, 0.0762)\n",
      "last loss 4.84973\n",
      "epoch 17 valid (0.19432616414294665, 0.0998)\n",
      "last loss 4.84056\n",
      "epoch 18 valid (0.19757946081407107, 0.2678)\n",
      "last loss 4.80868\n",
      "epoch 19 valid (0.19874509712515742, 0.3464)\n",
      "last loss 4.75974\n",
      "train (0.21151724760997723, 0.3338)\n",
      "test (0.19874509712515742, 0.3464)\n"
     ]
    }
   ],
   "source": [
    "def generate_data(corpus_sents, max_length, extend_vocabulary):\n",
    "    sen_en = np.full((len(corpus_sents), max_length), extend_vocabulary[\"<pad>\"], dtype=np.int32)\n",
    "    sen_de = np.full((len(corpus_sents), max_length), extend_vocabulary[\"<pad>\"], dtype=np.int32)\n",
    "    sen_en_length = np.zeros((len(corpus_sents),), dtype=np.int32)\n",
    "    sen_de_length = np.zeros((len(corpus_sents),), dtype=np.int32)\n",
    "\n",
    "    def get_random_sequence(sent, max_length):\n",
    "        x = np.full((max_length), extend_vocabulary[\"<pad>\"], dtype=np.int32)\n",
    "        for i, word in enumerate(sent):\n",
    "            if word in extend_vocabulary:\n",
    "                x[i] = extend_vocabulary[word]\n",
    "            else:\n",
    "                x[i] = extend_vocabulary[\"<unk>\"]\n",
    "        return x\n",
    "\n",
    "    for i in range(len(corpus_sents)):\n",
    "        l = min(len(corpus_sents[i]), max_length-2) + 2\n",
    "        sen_en[i, :] = get_random_sequence([\"<s>\"] + corpus_sents[i][:max_length-2] + [\"</s>\"], max_length)\n",
    "        sen_de[i, :l-1] = sen_en[i, 1:l]\n",
    "        sen_en_length[i] = l\n",
    "        sen_de_length[i] = l - 1\n",
    "    \n",
    "    return sen_en, sen_de, sen_en_length, sen_de_length\n",
    "\n",
    "def get_total_accuracy(data_sen_en, data_sen_de, data_sen_en_length, data_sen_de_length, extend_vocabulary, pretrained_lstm):\n",
    "    n_hit_word, n_hit_length = 0, 0\n",
    "    n_total_word, n_total_length = 0, 0\n",
    "    cur_idx = 0\n",
    "    while cur_idx < data_sen_en.shape[0]:\n",
    "        batch_sen_en = data_sen_en[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_de = data_sen_de[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_en_length = data_sen_en_length[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_de_length = data_sen_de_length[cur_idx: cur_idx + n_batch_size]\n",
    "        \n",
    "        _, predictions = pretrained_lstm.predict(\n",
    "            batch_sen_en, batch_sen_de, batch_sen_en_length, batch_sen_de_length\n",
    "        )\n",
    "        cur_idx += n_batch_size\n",
    "        cur_acc_word, cur_acc_length = evaluate(batch_sen_de, batch_sen_de_length, predictions, extend_vocabulary)\n",
    "        n_hit_word += cur_acc_word * np.sum(batch_sen_en_length)\n",
    "        n_total_word += np.sum(batch_sen_en_length)\n",
    "        n_hit_length += cur_acc_length * batch_sen_en.shape[0]\n",
    "        n_total_length += batch_sen_en.shape[0]\n",
    "    return 1. * n_hit_word / n_total_word, 1. * n_hit_length / n_total_length\n",
    "    \n",
    "# hyperparameter\n",
    "vocabulary_size = 20000\n",
    "origin_vocabulary = {}\n",
    "for word, n in corpus_counter.most_common(vocabulary_size):\n",
    "    origin_vocabulary[\"{}\".format(word)] = len(origin_vocabulary)\n",
    "extend_vocabulary = dict(origin_vocabulary)\n",
    "for w in [\"<pad>\", \"<unk>\", \"<s>\", \"</s>\"]:\n",
    "    extend_vocabulary[w] = len(extend_vocabulary)\n",
    "reverse_extend_vocabulary = {v: k for k, v in extend_vocabulary.items()}\n",
    "state_size=64\n",
    "n_max_length=100\n",
    "n_batch_size=30\n",
    "\n",
    "# generate training/testing data\n",
    "preprocess_sents = []\n",
    "for sent, _ in train_preprocess:\n",
    "    preprocess_sents.append(sent)\n",
    "n_train = int(len(preprocess_sents) * 0.8)\n",
    "n_valid = len(preprocess_sents) - n_train\n",
    "\n",
    "print(\"n_train\", n_train, \"n_valid\", n_valid)\n",
    "train_sen_en, train_sen_de, train_sen_en_length, train_sen_de_length = generate_data(preprocess_sents[:n_train],\n",
    "                                                                                     n_max_length, extend_vocabulary)\n",
    "valid_sen_en, valid_sen_de, valid_sen_en_length, valid_sen_de_length = generate_data(preprocess_sents[n_train:],\n",
    "                                                                                     n_max_length, extend_vocabulary)\n",
    "print(train_sen_en[0])\n",
    "print(preprocess_sents[0])\n",
    "print([reverse_extend_vocabulary[i] for i in train_sen_en[0]])\n",
    "print(valid_sen_en[0])\n",
    "print(preprocess_sents[n_train])\n",
    "print([reverse_extend_vocabulary[i] for i in valid_sen_en[0]])\n",
    "\n",
    "pretrained_lstm = EncoderDecoder(vocabulary=extend_vocabulary, state_size=state_size, n_max_length=n_max_length)\n",
    "\n",
    "for epoch in range(20):\n",
    "    cur_idx = 0\n",
    "    while cur_idx < train_sen_en.shape[0]:\n",
    "        batch_sen_en = train_sen_en[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_de = train_sen_de[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_en_length = train_sen_en_length[cur_idx: cur_idx + n_batch_size]\n",
    "        batch_sen_de_length = train_sen_de_length[cur_idx: cur_idx + n_batch_size]\n",
    "        \n",
    "        loss, predictions, sen_en_embedding, mask, cross_ent, clipped_gradients = pretrained_lstm.train(\n",
    "            batch_sen_en, batch_sen_de, batch_sen_en_length, batch_sen_de_length\n",
    "        )\n",
    "        cur_idx += n_batch_size\n",
    "    print(\"epoch\", epoch, \"valid\", get_total_accuracy(\n",
    "        valid_sen_en, valid_sen_de, valid_sen_en_length, valid_sen_de_length, extend_vocabulary, pretrained_lstm\n",
    "    )) \n",
    "    print(\"last loss\", loss)\n",
    "print(\"train\", get_total_accuracy(\n",
    "    train_sen_en, train_sen_de, train_sen_en_length, train_sen_de_length, extend_vocabulary, pretrained_lstm\n",
    "))\n",
    "print(\"test\", get_total_accuracy(\n",
    "    valid_sen_en, valid_sen_de, valid_sen_en_length, valid_sen_de_length, extend_vocabulary, pretrained_lstm\n",
    ")) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}