short_txt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 20114: expected 15 fields, saw 22\\nSkipping line 36320: expected 15 fields, saw 22\\nSkipping line 38067: expected 15 fields, saw 22\\nSkipping line 40844: expected 15 fields, saw 22\\nSkipping line 111102: expected 15 fields, saw 22\\nSkipping line 128910: expected 15 fields, saw 22\\n'\n"
     ]
    }
   ],
   "source": [
    "reviews = pd.read_table(\"amazon_data/sampled_reviews_small.tsv\",error_bad_lines=False,low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['marketplace', 'customer_id', 'review_id', 'product_id',\n",
       "       'product_parent', 'product_title', 'product_category', 'star_rating',\n",
       "       'helpful_votes', 'total_votes', 'vine', 'verified_purchase',\n",
       "       'review_headline', 'review_body', 'review_date'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'product_category'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m~/akirato/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2896\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2897\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'product_category'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-fb4f169a05d8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnum_prod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreviews\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'product_category'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/akirato/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2993\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2994\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2995\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2996\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2997\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/akirato/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2897\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2898\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2899\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2900\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2901\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'product_category'"
     ]
    }
   ],
   "source": [
    "num_prod = len(reviews['product_category'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "53"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_prod"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "from keras.layers import Input, LSTM, Dense\n",
    "import numpy as np\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "from scipy.sparse import csr_matrix\n",
    "from gensim.models import Word2Vec\n",
    "import logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "_BOS = \"_BOS \"\n",
    "_EOS = \" _EOS\"\n",
    "DIM_SIZE = 300\n",
    "w2v_model = Word2Vec.load(\"amazon_data/model.en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 128328: expected 15 fields, saw 22\\nSkipping line 134749: expected 15 fields, saw 22\\nSkipping line 174759: expected 15 fields, saw 22\\nSkipping line 197766: expected 15 fields, saw 22\\nSkipping line 202569: expected 15 fields, saw 22\\nSkipping line 382089: expected 15 fields, saw 22\\nSkipping line 397816: expected 15 fields, saw 22\\nSkipping line 398935: expected 15 fields, saw 22\\nSkipping line 416875: expected 15 fields, saw 22\\nSkipping line 431901: expected 15 fields, saw 22\\nSkipping line 554909: expected 15 fields, saw 22\\nSkipping line 657304: expected 15 fields, saw 22\\nSkipping line 691801: expected 15 fields, saw 22\\nSkipping line 711377: expected 15 fields, saw 22\\nSkipping line 741194: expected 15 fields, saw 22\\nSkipping line 772352: expected 15 fields, saw 22\\nSkipping line 775569: expected 15 fields, saw 22\\nSkipping line 778834: expected 15 fields, saw 22\\nSkipping line 779740: expected 15 fields, saw 22\\nSkipping line 789067: expected 15 fields, saw 22\\nSkipping line 856382: expected 15 fields, saw 22\\nSkipping line 857851: expected 15 fields, saw 22\\nSkipping line 897605: expected 15 fields, saw 22\\nSkipping line 899032: expected 15 fields, saw 22\\nSkipping line 904965: expected 15 fields, saw 22\\nSkipping line 925766: expected 15 fields, saw 22\\nSkipping line 980954: expected 15 fields, saw 22\\nSkipping line 1076071: expected 15 fields, saw 22\\nSkipping line 1107836: expected 15 fields, saw 22\\nSkipping line 1234287: expected 15 fields, saw 22\\nSkipping line 1237668: expected 15 fields, saw 22\\nSkipping line 1238947: expected 15 fields, saw 22\\nSkipping line 1269005: expected 15 fields, saw 22\\nSkipping line 1290341: expected 15 fields, saw 22\\nSkipping line 1299988: expected 15 fields, saw 22\\nSkipping line 1429907: expected 15 fields, saw 22\\nSkipping line 1448007: expected 15 fields, saw 22\\n'\n"
     ]
    }
   ],
   "source": [
    "def remove_punct(sentence):\n",
    "    #sentence = re.sub(stop, \" \", sentence) \n",
    "    return sentence.lower().translate(str.maketrans('', '', string.punctuation))\n",
    "\n",
    "def create_dictionary(data_path):\n",
    "    reviews = pd.read_table(data_path,error_bad_lines=False,low_memory=False)\n",
    "\n",
    "    input_texts = [] \n",
    "    target_texts = []\n",
    "    prod_category = []\n",
    "    # idx2word\n",
    "    input_title_vocab = set()\n",
    "    target_review_vocab = set()\n",
    "    #\n",
    "    for index,row in reviews.iterrows():\n",
    "        attributes = [row[\"product_title\"],row[\"review_headline\"],row[\"review_body\"]]\n",
    "        if not isinstance(row[\"product_title\"],str): \n",
    "            continue\n",
    "        if not isinstance(row[\"review_headline\"],str): \n",
    "            continue\n",
    "        if not isinstance(row[\"review_body\"],str): \n",
    "            continue   \n",
    "        input_text, target_text = remove_punct(row[\"product_title\"]),\\\n",
    "        remove_punct(\" \".join([row[\"review_headline\"],row[\"review_body\"]]))\n",
    "        target_text = _BOS + target_text + _EOS\n",
    "        input_texts.append(input_text)\n",
    "        target_texts.append(target_text)\n",
    "        prod_category.append(row[\"product_category\"])\n",
    "        for word in input_text.split():\n",
    "            if word not in input_title_vocab:\n",
    "                input_title_vocab.add(word)\n",
    "        for word in target_text.split():\n",
    "            if word not in target_review_vocab:\n",
    "                target_review_vocab.add(word)\n",
    "\n",
    "    return input_texts, target_texts, input_title_vocab, target_review_vocab, prod_category\n",
    "\n",
    "input_texts, target_texts, input_title_vocab, target_review_vocab, prod_category = create_dictionary(\"amazon_data/sampled_reviews.tsv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = sorted(list(input_title_vocab.union(target_review_vocab)))\n",
    "target_review_vocab = sorted(list(target_review_vocab))\n",
    "# the number of sample vocablary\n",
    "encoder_vocab_size = len(input_title_vocab)\n",
    "decoder_vocab_size = len(target_review_vocab)\n",
    "vocab_size = len(vocab)\n",
    "# Define max length of encoder / decoder\n",
    "#max_encoder_seq_length = max([len(text.split()) for text in input_texts])\n",
    "#max_decoder_seq_length = max([len(text.split()) for text in target_texts])\n",
    "input_di_text,target_di_text = {},{}\n",
    "text_index = 0\n",
    "max_seq_length = 0\n",
    "for text in input_texts:\n",
    "    if len(text.split()) <= 100:\n",
    "        input_di_text[text_index] = text\n",
    "        text_index += 1\n",
    "        max_seq_length = max(max_seq_length,len(text.split()))\n",
    "for text in target_texts:\n",
    "    if len(text.split()) <= 100:\n",
    "        target_di_text[text_index] = text\n",
    "        text_index += 1\n",
    "        max_seq_length = max(max_seq_length,len(text.split()))\n",
    "    \n",
    "\n",
    "num_prod = len(reviews['product_category'].unique())\n",
    "inverse_input_vocab = dict(\n",
    "    [(word, id) for id, word in enumerate(input_title_vocab)])\n",
    "inverse_target_vocab = dict(\n",
    "    [(word, id) for id, word in enumerate(target_review_vocab)])\n",
    "inverse_vocab = dict(\n",
    "    [(word, id) for id, word in enumerate(vocab)])\n",
    "inverse_prod_category = dict(\n",
    "    [(word, id) for id, word in enumerate(reviews['product_category'].unique())])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.callbacks import TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate array with shape (1520031, 100, 600) and data type float32",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-63-f8b02921f291>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m encoder_input_data = np.zeros(\n\u001b[1;32m      2\u001b[0m     \u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_di_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_seq_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDIM_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     dtype='float32')\n\u001b[0m\u001b[1;32m      4\u001b[0m decoder_input_data = np.zeros(\n\u001b[1;32m      5\u001b[0m     \u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_di_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_seq_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDIM_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mMemoryError\u001b[0m: Unable to allocate array with shape (1520031, 100, 600) and data type float32"
     ]
    }
   ],
   "source": [
    "encoder_input_data = np.zeros(\n",
    "    (len(input_di_text), max_seq_length, DIM_SIZE),\n",
    "    dtype='float32')\n",
    "decoder_input_data = np.zeros(\n",
    "    (len(target_di_text), max_seq_length, DIM_SIZE),\n",
    "   dtype='float32')\n",
    "decoder_target_data = np.zeros(\n",
    "    (len(target_di_text), max_seq_length, DIM_SIZE),\n",
    "    dtype='float32')\n",
    "prod_category_data = np.zeros(\n",
    "    (len(prod_category), 1, num_prod),\n",
    "    dtype='float32')\n",
    "\n",
    "for pair_text_idx, (input_text, target_text,prod_cat) in enumerate(zip(input_texts, target_texts,prod_category)):\n",
    "    for timestep, word in enumerate(input_text.split()):\n",
    "        encoder_input_data[pair_text_idx, timestep, inverse_vocab[word]] = 1.\n",
    "    # decoder_target_data is ahead of decoder_input_data by one timestep\n",
    "    for timestep, word in enumerate(target_text.split()):\n",
    "        decoder_input_data[pair_text_idx, timestep, inverse_vocab[word]] = 1.\n",
    "        if timestep > 0:\n",
    "            # decoder_target_data will be ahead by one timestep（LSTM)\n",
    "            # decoder_target_data will not include the start character.\n",
    "            decoder_target_data[pair_text_idx, timestep - 1, inverse_vocab[word]] = 1.\n",
    "    prod_category_data[pair_text_idx,0, inverse_prod_category[prod_cat]] = 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "NUM_HIDDEN_UNITS = 128 # NUM_HIDDEN_LAYERS\n",
    "BATCH_SIZE = 512\n",
    "NUM_EPOCHS = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "class Attention(tf.keras.Model):\n",
    "\n",
    "    def __init__(self, units):\n",
    "        # initialize initial values\n",
    "        super(Attention, self).__init__()\n",
    "        # architecture of nn\n",
    "        self.W1 = tf.keras.layers.Dense(units)\n",
    "        self.W2 = tf.keras.layers.Dense(units)\n",
    "        self.V = tf.keras.layers.Dense(1)\n",
    "\n",
    "    def call(self, features, hudden):\n",
    "        hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
    "        prob_score = tf.nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis))\n",
    "        attention_weights = tf.nn.softmax(self.V(prob_score), axis=1)\n",
    "        context_vector = attention_weights * features\n",
    "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "        return context_vector, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_texts = list(input_di_text.values())\n",
    "target_texts = list(target_di_text.values())\n",
    "def batch_generator(batch_size):\n",
    "    while True:\n",
    "        encoder_input_data = np.zeros(\n",
    "            (batch_size, max_seq_length, DIM_SIZE),\n",
    "            dtype='float32')\n",
    "        decoder_input_data = np.zeros(\n",
    "            (batch_size, max_seq_length, DIM_SIZE),\n",
    "            dtype='float32')\n",
    "        decoder_target_data = np.zeros(\n",
    "            (batch_size, max_seq_length, DIM_SIZE),\n",
    "            dtype='float32')\n",
    "        prod_category_data = np.zeros(\n",
    "            (batch_size, 1, num_prod),\n",
    "            dtype='float32')\n",
    "        n_batches_for_epoch = len(input_di_text)//batch_size\n",
    "        for i in range(n_batches_for_epoch):\n",
    "            for pair_text_idx, (input_text, target_text,prod_cat) in enumerate(zip(input_texts[batch_size*i:batch_size*(i+1)], target_texts[batch_size*i:batch_size*(i+1)],prod_category[batch_size*i:batch_size*(i+1)])):\n",
    "                for timestep, word in enumerate(input_text.split()):\n",
    "                    if word in w2v_model.wv:\n",
    "                        encoder_input_data[pair_text_idx, timestep] = w2v_model.wv[word]\n",
    "                # decoder_target_data is ahead of decoder_input_data by one timestep\n",
    "                for timestep, word in enumerate(target_text.split()):\n",
    "                    if word in w2v_model.wv:\n",
    "                        decoder_input_data[pair_text_idx, timestep] = w2v_model.wv[word]\n",
    "                    if timestep > 0:\n",
    "                        # decoder_target_data will be ahead by one timestep（LSTM)\n",
    "                        decoder_target_data[pair_text_idx, timestep - 1] = w2v_model.wv[word]\n",
    "                prod_category_data[pair_text_idx,0, inverse_prod_category[prod_cat]] = 1.\n",
    "            yield([encoder_input_data, decoder_input_data],[decoder_target_data,prod_category_data])\n",
    "#         for i in range(n_batches_for_epoch):\n",
    "#             index_batch = range(X.shape[0])[batch_size*i:batch_size*(i+1)]\n",
    "#             X_batch = X[index_batch,:].todense()\n",
    "#             yield(X_batch, X_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_18\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_36 (InputLayer)           (None, None, 300)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_37 (InputLayer)           (None, None, 300)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_35 (LSTM)                  [(None, 128), (None, 219648      input_36[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "lstm_36 (LSTM)                  [(None, None, 128),  219648      input_37[0][0]                   \n",
      "                                                                 lstm_35[0][1]                    \n",
      "                                                                 lstm_35[0][2]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_52 (Dense)                (None, None, 300)    38700       lstm_36[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_53 (Dense)                (None, None, 100)    30100       dense_52[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_54 (Dense)                (None, None, 53)     5353        dense_53[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 513,449\n",
      "Trainable params: 513,449\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/reddy/akirato/lib/python3.6/site-packages/ipykernel_launcher.py:38: UserWarning: The semantics of the Keras 2 argument `steps_per_epoch` is not the same as the Keras 1 argument `samples_per_epoch`. `steps_per_epoch` is the number of batches to draw from the generator at each epoch. Basically steps_per_epoch = samples_per_epoch/batch_size. Similarly `nb_val_samples`->`validation_steps` and `val_samples`->`steps` arguments have changed. Update your method calls accordingly.\n",
      "/home/reddy/akirato/lib/python3.6/site-packages/ipykernel_launcher.py:38: UserWarning: Update your `fit_generator` call to the Keras 2 API: `fit_generator(generator=<generator..., shuffle=True, callbacks=[<keras.ca..., verbose=True, steps_per_epoch=1520031, epochs=10)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "    238/1520031 [..............................] - ETA: 151:15:19 - loss: -1024.2956 - dense_52_loss: -1024.5825 - dense_54_loss: 0.2870"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Encoder Architecture\n",
    "\"\"\"\n",
    "\n",
    "encoder_inputs = Input(shape=(None, DIM_SIZE))\n",
    "encoder_lstm = LSTM(units=NUM_HIDDEN_UNITS, return_state=True)\n",
    "# x-axis: time-step lstm\n",
    "encoder_outputs, state_h, state_c = encoder_lstm(encoder_inputs)\n",
    "encoder_states = [state_h, state_c] # We discard `encoder_outputs` and only keep the states.\n",
    "\n",
    "\"\"\"\n",
    "Decoder Architecture\n",
    "\"\"\"\n",
    "# We set up our decoder to return full output sequences,\n",
    "# and to return internal states as well. We don't use the\n",
    "# return states in the training model, but we will use them in inference.\n",
    "decoder_inputs = Input(shape=(None, DIM_SIZE))\n",
    "decoder_lstm = LSTM(units=NUM_HIDDEN_UNITS, return_sequences=True, return_state=True)\n",
    "# x-axis: time-step lstm\n",
    "decoder_outputs, de_state_h, de_state_c = decoder_lstm(decoder_inputs, initial_state=encoder_states) # Set up the decoder, using `encoder_states` as initial state.\n",
    "decoder_softmax_layer = Dense(DIM_SIZE, activation='softmax')\n",
    "decoder_outputs = decoder_softmax_layer(decoder_outputs)\n",
    "\n",
    "classification_dense = Dense(100, activation = 'relu')(decoder_outputs)\n",
    "classification_outputs = Dense(num_prod,activation = 'relu')(classification_dense)\n",
    "\"\"\"\n",
    "Encoder-Decoder Architecture\n",
    "\"\"\"\n",
    "# Define the model that will turn, `encoder_input_data` & `decoder_input_data` into `decoder_target_data`\n",
    "model = Model([encoder_inputs, decoder_inputs], [decoder_outputs,classification_outputs])\n",
    "print(model.summary())\n",
    "model.compile(optimizer=\"rmsprop\", loss=[\"categorical_crossentropy\",\"binary_crossentropy\"]) # Set up model\n",
    "model.fit_generator(generator=batch_generator(BATCH_SIZE),\n",
    "                    nb_epoch=10,\n",
    "                    shuffle=True,\n",
    "                    samples_per_epoch=len(input_di_text),\n",
    "                    callbacks=[TensorBoard(log_dir='/tmp/autoencoder')],\n",
    "                    verbose=True)\n",
    "\n",
    "# model.fit(x=[encoder_input_data, decoder_input_data], y=[decoder_target_data,prod_category_data],\n",
    "#           batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, validation_split=0.2) # Run training\n",
    "model.save(\"seq2seq_translate_model.txt\") # Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# inputs=encoder_inputs, outputs=encoder_states\n",
    "encoder_model = Model(inputs=encoder_inputs, outputs=encoder_states)\n",
    "\n",
    "# State from encoder\n",
    "decoder_state_input_h = Input(shape=(NUM_HIDDEN_UNITS,))\n",
    "decoder_state_input_c = Input(shape=(NUM_HIDDEN_UNITS,))\n",
    "decoder_state_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "# x-axis: time-step lstm\n",
    "decoder_outputs, state_h, state_c = decoder_lstm(decoder_inputs, initial_state=decoder_state_inputs)\n",
    "decoder_states = [state_h, state_c]\n",
    "decoder_outputs = decoder_softmax_layer(decoder_outputs)\n",
    "decoder_model = Model(inputs=[decoder_inputs] + decoder_state_inputs, outputs=[decoder_outputs] + decoder_states)\n",
    "\n",
    "\n",
    "\n",
    "input_idx2word = dict(\n",
    "    (id, word) for word, id in inverse_vocab.items())\n",
    "target_idx2word = dict(\n",
    "    (id, word) for word, id in inverse_vocab.items())\n",
    "\n",
    "def decode_sequence(input_seq):\n",
    "    # Encode the input as state vectors.\n",
    "    states_value = encoder_model.predict(input_seq)\n",
    "    # Generate empty target sequence of length 1.\n",
    "    target_seq = np.zeros((1, 1, vocab_size))\n",
    "    # Populate the first character of target sequence with the start character.\n",
    "    #target_seq[0, 0, target_token_index['\\t']] = 1.\n",
    "\n",
    "    # Sampling loop for a batch of sequences\n",
    "    # (to simplify, here we assume a batch of size 1).\n",
    "    stop_condition = False\n",
    "    decoded_sentence = \"\"\n",
    "    while not stop_condition:\n",
    "        output_tokens, h, c = decoder_model.predict(\n",
    "            [target_seq] + states_value)\n",
    "\n",
    "        # Sample a token\n",
    "        sampled_token_index = np.argmax(output_tokens[0, -1, :])\n",
    "        sampled_char = target_idx2word[sampled_token_index]\n",
    "        decoded_sentence += \" \"+sampled_char\n",
    "\n",
    "        # Exit condition: either hit max length\n",
    "        # or find stop character.\n",
    "        if (sampled_char == '\\n' or\n",
    "           len(decoded_sentence) > max_seq_length):\n",
    "            stop_condition = True\n",
    "        num_decoder_tokens = vocab_size\n",
    "        # Update the target sequence (of length 1).\n",
    "        target_seq = np.zeros((1, 1, num_decoder_tokens))\n",
    "        target_seq[0, 0, sampled_token_index] = 1.\n",
    "\n",
    "        # Update states\n",
    "        states_value = [h, c]\n",
    "\n",
    "    return decoded_sentence\n",
    "\n",
    "def classify_sequence(input_seq,decode_seq):\n",
    "    print(input_seq.shape,decode_seq.shape)\n",
    "    return model.predict([input_seq,decode_seq])\n",
    "\n",
    "for seq_index in range(5):\n",
    "    # Take one sequence (part of the training set) for trying out decoding.\n",
    "    input_seq = encoder_input_data[seq_index: seq_index + 1]\n",
    "    decode_seq = decoder_input_data[seq_index:seq_index + 1]\n",
    "    decoded_sentence = decode_sequence(input_seq)\n",
    "\n",
    "    print('-')\n",
    "    print('Input sentence:', input_texts[seq_index])\n",
    "    print('Decoded sentence:', decoded_sentence)\n",
    "    output = classify_sequence(input_seq, decode_seq)\n",
    "    print(\"classifier:\", np.argmax(output[1][0, -1, :]),[x.shape for x in output])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inverse_target_vocab.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}