pipe_PosAndNeg.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T01:57:34.411966Z",
     "start_time": "2018-08-10T01:57:32.663999Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/deep/pythonenv/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras as K\n",
    "import numpy as np\n",
    "import os\n",
    "from keras.utils import to_categorical\n",
    "import _pickle as pickle\n",
    "from pathlib import Path\n",
    "from models import *\n",
    "from datautils import *\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T01:57:34.718352Z",
     "start_time": "2018-08-10T01:57:34.436025Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "session = tf.Session(config=config)\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T01:57:37.685340Z",
     "start_time": "2018-08-10T01:57:34.769710Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_embed_weight=np.load('./cache/word_embed.npy')\n",
    "train_text = np.load('./cache/train_text.npy')\n",
    "train_label = np.load('./cache/train_label.npy')\n",
    "val_y = np.load('./cache/val_label.npy')\n",
    "val_X = np.load('./cache/val_X.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T01:57:37.760480Z",
     "start_time": "2018-08-10T01:57:37.740052Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((130687,), (18669,))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_label.shape, val_y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Divide Pos and Neg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T01:57:37.943675Z",
     "start_time": "2018-08-10T01:57:37.926772Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((111741,), (15975,))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_PN_idx = (train_label != 2)\n",
    "train_PN_label = train_label[train_PN_idx]\n",
    "val_PN_idx = (val_y != 2)\n",
    "val_PN_label = val_y[val_PN_idx]\n",
    "train_PN_label.shape, val_PN_label.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T01:57:41.015615Z",
     "start_time": "2018-08-10T01:57:39.446670Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_text = train_text[train_PN_idx]\n",
    "train_label = train_PN_label < 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T01:57:41.119011Z",
     "start_time": "2018-08-10T01:57:41.070149Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_X = val_X[val_PN_idx]\n",
    "val_y = to_categorical(val_PN_label < 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T01:58:54.599012Z",
     "start_time": "2018-08-10T01:58:54.583218Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((111741,), (15975,), (111741,), (15975, 2))"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_text.shape, val_X.shape, train_label.shape, val_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T01:57:42.347000Z",
     "start_time": "2018-08-10T01:57:41.959306Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pos:  81222\n",
      "Neg:  30519\n"
     ]
    }
   ],
   "source": [
    "print('Pos: ', sum(train_label==0))\n",
    "print('Neg: ', sum(train_label==1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T01:57:43.486203Z",
     "start_time": "2018-08-10T01:57:43.470648Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_nb = val_y.shape[1]\n",
    "class_nb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T01:58:46.582987Z",
     "start_time": "2018-08-10T01:58:46.119342Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_text = np.load('./cache/val_text.npy')\n",
    "val_X = val_text[val_PN_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-10T02:09:06.046695Z",
     "start_time": "2018-08-10T02:08:49.533900Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('./cache/PN_train_text.npy',train_text)\n",
    "np.save('./cache/PN_train_label.npy', train_label)\n",
    "np.save('./cache/PN_val_X.npy', val_X)\n",
    "np.save('./cache/PN_val_y.npy', val_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TextCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:10:08.876036Z",
     "start_time": "2018-08-09T09:10:08.292992Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 150, 256)     11369216    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 150, 256)     65792       embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 150, 256)     1024        time_distributed_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 150, 256)     0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 148, 128)     98432       activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 147, 128)     131200      activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 146, 128)     163968      activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 148, 128)     512         conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 147, 128)     512         conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 146, 128)     512         conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 148, 128)     0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 147, 128)     0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 146, 128)     0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 128)          0           activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 128)          0           activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_3 (GlobalM (None, 128)          0           activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_feat (Concatenate)         (None, 384)          0           global_max_pooling1d_1[0][0]     \n",
      "                                                                 global_max_pooling1d_2[0][0]     \n",
      "                                                                 global_max_pooling1d_3[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 384)          0           conv_feat[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 300)          115500      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 300)          1200        dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 300)          0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 300)          0           activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 2)            602         dropout_2[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 11,948,470\n",
      "Trainable params: 577,374\n",
      "Non-trainable params: 11,371,096\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "batch_size=128\n",
    "model_name='textcnn'\n",
    "model=get_textcnn_v2(sentence_length, word_embed_weight, class_nb)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:10:13.528929Z",
     "start_time": "2018-08-09T09:10:13.514059Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_batch_generator = word_text_train_batch_generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:10:57.483914Z",
     "start_time": "2018-08-09T09:10:25.659295Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 31s 35ms/step - loss: 0.4125 - acc: 0.8115 - val_loss: 0.3541 - val_acc: 0.8416\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'val_seq' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-8ad485d82030>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m     model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n\u001b[1;32m      3\u001b[0m                     epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_seq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0mpre\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mrec\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mf1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscore\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_label\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'val_seq' is not defined"
     ]
    }
   ],
   "source": [
    "for i in range(3):\n",
    "    model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "    pred = np.squeeze(model.predict(val_X))\n",
    "    pre,rec,f1 = score(pred, val_y)\n",
    "    print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:12:05.656853Z",
     "start_time": "2018-08-09T09:11:35.299324Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 29s 34ms/step - loss: 0.3564 - acc: 0.8415 - val_loss: 0.3583 - val_acc: 0.8347\n",
      "0.6750048856752003 0.7793321299638989 0.7234265368101372\n"
     ]
    }
   ],
   "source": [
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "pred = np.squeeze(model.predict(val_X))\n",
    "pre,rec,f1 = score(pred, val_y)\n",
    "print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:12:36.540426Z",
     "start_time": "2018-08-09T09:12:05.711516Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 30s 34ms/step - loss: 0.3414 - acc: 0.8484 - val_loss: 0.3315 - val_acc: 0.8553\n",
      "0.7560386473429952 0.7062274368231047 0.730284647690154\n"
     ]
    }
   ],
   "source": [
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "pred = np.squeeze(model.predict(val_X))\n",
    "pre,rec,f1 = score(pred, val_y)\n",
    "print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:13:42.280572Z",
     "start_time": "2018-08-09T09:13:10.353201Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 31s 35ms/step - loss: 0.2698 - acc: 0.8863 - val_loss: 0.3143 - val_acc: 0.8645\n",
      "0.7538633818589026 0.7594765342960289 0.7566595481623019\n"
     ]
    }
   ],
   "source": [
    "K.set_value(model.optimizer.lr, 0.0001)\n",
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "pred = np.squeeze(model.predict(val_X))\n",
    "pre,rec,f1 = score(pred, val_y)\n",
    "print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:13:10.298917Z",
     "start_time": "2018-08-09T09:12:36.594080Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 31s 36ms/step - loss: 0.3277 - acc: 0.8560 - val_loss: 0.3182 - val_acc: 0.8601\n",
      "0.7606168446026097 0.7233754512635379 0.7415288539377818\n"
     ]
    }
   ],
   "source": [
    "model.layers[1].trainable = True\n",
    "model.compile(loss='categorical_crossentropy',optimizer=\"adam\",metrics=['accuracy'])\n",
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "pred = np.squeeze(model.predict(val_X))\n",
    "pre,rec,f1 = score(pred, val_y)\n",
    "print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:14:50.056483Z",
     "start_time": "2018-08-09T09:14:19.572309Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 29s 34ms/step - loss: 0.2463 - acc: 0.8968 - val_loss: 0.3227 - val_acc: 0.8617\n",
      "0.7357809847198642 0.7822653429602888 0.7583114610673666\n"
     ]
    }
   ],
   "source": [
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "pred = np.squeeze(model.predict(val_X))\n",
    "pre,rec,f1 = score(pred, val_y)\n",
    "print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:03:51.266928Z",
     "start_time": "2018-08-09T09:02:27.441707Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 82s 94ms/step - loss: 0.2290 - acc: 0.9032 - val_loss: 0.3830 - val_acc: 0.8400\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f559465c320>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[1].trainable = True\n",
    "model.compile(loss='categorical_crossentropy',optimizer=\"adam\",metrics=['accuracy'])\n",
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T10:34:50.224745Z",
     "start_time": "2018-08-09T10:34:49.688876Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 150)               0         \n",
      "_________________________________________________________________\n",
      "embedding_2 (Embedding)      (None, 150, 256)          11369216  \n",
      "_________________________________________________________________\n",
      "time_distributed_2 (TimeDist (None, 150, 256)          65792     \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 150, 256)          1024      \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 150, 256)          0         \n",
      "_________________________________________________________________\n",
      "gru_2 (GRU)                  (None, 64)                61632     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 256)               16640     \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 2)                 514       \n",
      "=================================================================\n",
      "Total params: 11,515,842\n",
      "Trainable params: 145,602\n",
      "Non-trainable params: 11,370,240\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "batch_size=128\n",
    "model_name='textrnn'\n",
    "model=get_rnn(sentence_length, word_embed_weight, class_nb)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T10:34:50.532083Z",
     "start_time": "2018-08-09T10:34:50.516279Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_batch_generator = word_text_train_batch_generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T10:40:23.926105Z",
     "start_time": "2018-08-09T10:34:52.684613Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 97s 111ms/step - loss: 0.5661 - acc: 0.7358 - val_loss: 0.3954 - val_acc: 0.8219\n",
      "0.6758253933082207 0.6881768953068592 0.6819452207937394\n",
      "Epoch 1/1\n",
      "872/872 [==============================] - 96s 110ms/step - loss: 0.3630 - acc: 0.8388 - val_loss: 0.3426 - val_acc: 0.8506\n",
      "0.7541635595326871 0.6845667870036101 0.7176818450620935\n",
      "Epoch 1/1\n",
      "872/872 [==============================] - 96s 111ms/step - loss: 0.3336 - acc: 0.8527 - val_loss: 0.3282 - val_acc: 0.8575\n",
      "0.7909802862543883 0.6608754512635379 0.7200983405039951\n"
     ]
    }
   ],
   "source": [
    "for i in range(3):\n",
    "    model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "    pred = np.squeeze(model.predict(val_X))\n",
    "    pre,rec,f1 = score(pred, val_y)\n",
    "    print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T10:42:20.443490Z",
     "start_time": "2018-08-09T10:40:29.819782Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 97s 111ms/step - loss: 0.3083 - acc: 0.8664 - val_loss: 0.3245 - val_acc: 0.8575\n",
      "0.7286229577763632 0.7748194945848376 0.7510114816839805\n"
     ]
    }
   ],
   "source": [
    "K.set_value(model.optimizer.lr, 0.0001)\n",
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "pred = np.squeeze(model.predict(val_X))\n",
    "pre,rec,f1 = score(pred, val_y)\n",
    "print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T10:44:35.660790Z",
     "start_time": "2018-08-09T10:42:46.756627Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 95s 109ms/step - loss: 0.3002 - acc: 0.8694 - val_loss: 0.3223 - val_acc: 0.8567\n",
      "0.7261025532812829 0.7763989169675091 0.7504088976120379\n"
     ]
    }
   ],
   "source": [
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "pred = np.squeeze(model.predict(val_X))\n",
    "pre,rec,f1 = score(pred, val_y)\n",
    "print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T10:46:36.105431Z",
     "start_time": "2018-08-09T10:44:46.853564Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 96s 110ms/step - loss: 0.2958 - acc: 0.8714 - val_loss: 0.3197 - val_acc: 0.8592\n",
      "0.7353892602976062 0.769404332129964 0.7520123497629287\n"
     ]
    }
   ],
   "source": [
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "pred = np.squeeze(model.predict(val_X))\n",
    "pre,rec,f1 = score(pred, val_y)\n",
    "print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:33:40.292856Z",
     "start_time": "2018-08-09T09:33:40.277836Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sen_seq_length = 10\n",
    "word_seq_length = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:33:53.687952Z",
     "start_time": "2018-08-09T09:33:41.020540Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_text = np.load('./cache/val_text.npy')\n",
    "val_X = word_han_preprocess(val_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:33:53.820113Z",
     "start_time": "2018-08-09T09:33:53.799629Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_X = val_X[val_PN_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:33:55.076474Z",
     "start_time": "2018-08-09T09:33:53.930794Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 10, 15)            0         \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, 10, 256)           11665167  \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 10, 256)           295680    \n",
      "_________________________________________________________________\n",
      "attention_2 (Attention)      (None, 256)               266       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 2)                 514       \n",
      "=================================================================\n",
      "Total params: 12,028,443\n",
      "Trainable params: 12,027,931\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "train_batch_generator = word_han_train_batch_generator\n",
    "batch_size=128\n",
    "model_name='han'\n",
    "model=get_han(sen_seq_length, word_seq_length, word_embed_weight, class_nb)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:36:54.023442Z",
     "start_time": "2018-08-09T09:33:57.205874Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 83s 95ms/step - loss: 0.3784 - acc: 0.8309 - val_loss: 0.4026 - val_acc: 0.8293\n",
      "0.8347074990184531 0.4796931407942238 0.6092563404499213\n",
      "Epoch 1/1\n",
      "872/872 [==============================] - 82s 93ms/step - loss: 0.3111 - acc: 0.8658 - val_loss: 0.3546 - val_acc: 0.8475\n",
      "0.7430589381393083 0.6884025270758123 0.7146872803935347\n"
     ]
    }
   ],
   "source": [
    "for i in range(2):\n",
    "    model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "    pred = np.squeeze(model.predict(val_X))\n",
    "    pre,rec,f1 = score(pred, val_y)\n",
    "    print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:40:22.847703Z",
     "start_time": "2018-08-09T09:37:30.149615Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "872/872 [==============================] - 81s 93ms/step - loss: 0.2470 - acc: 0.8962 - val_loss: 0.3699 - val_acc: 0.8441\n",
      "0.719385449615906 0.7184115523465704 0.7188981711447279\n",
      "Epoch 1/1\n",
      "872/872 [==============================] - 82s 94ms/step - loss: 0.1827 - acc: 0.9256 - val_loss: 0.4817 - val_acc: 0.8344\n",
      "0.7890650274991912 0.5503158844765343 0.6484115379502858\n"
     ]
    }
   ],
   "source": [
    "for i in range(2):\n",
    "    model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "    pred = np.squeeze(model.predict(val_X))\n",
    "    pre,rec,f1 = score(pred, val_y)\n",
    "    print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-09T09:31:59.279364Z",
     "start_time": "2018-08-09T09:28:24.158Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "K.set_value(model.optimizer.lr, 0.0001)\n",
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))\n",
    "pred = np.squeeze(model.predict(val_X))\n",
    "pre,rec,f1 = score(pred, val_y)\n",
    "print(pre,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2018-08-09T09:04:01.118Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "756/872 [=========================>....] - ETA: 10s - loss: 0.1949 - acc: 0.9184"
     ]
    }
   ],
   "source": [
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}