pipeline_classifier5.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:47:31.126324Z",
     "start_time": "2018-08-07T13:47:29.209286Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import keras as K\n",
    "import numpy as np\n",
    "import os\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:47:31.134145Z",
     "start_time": "2018-08-07T13:47:31.132598Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import _pickle as pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T12:32:57.628606Z",
     "start_time": "2018-08-07T12:32:57.626887Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:47:33.784744Z",
     "start_time": "2018-08-07T13:47:31.903842Z"
    }
   },
   "outputs": [],
   "source": [
    "from models import *\n",
    "from datautils import *\n",
    "import tensorflow as tf\n",
    "\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "session = tf.Session(config=config)\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec\n",
    "from gensim.models.word2vec import LineSentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T03:52:33.130893Z",
     "start_time": "2018-08-07T03:52:32.905867Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "file = './data/corpus.csv'\n",
    "with open(file, 'r') as f:\n",
    "    t = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T03:52:34.052839Z",
     "start_time": "2018-08-07T03:52:33.941319Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "text = []\n",
    "for line in t:\n",
    "    text.append(line.split('\\t')[0])\n",
    "s = \"\\n\".join(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T03:52:35.091216Z",
     "start_time": "2018-08-07T03:52:35.084348Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'内测玩到现在，不管怎么样，五星\\n游戏的新手指印和可玩模式有欠缺\\n好玩 估计可以玩很多天不会腻~\\n四月末了，是不是要改成五月份呀\\n什么都好，就是电池太难耍刷到了\\n具体是4月几号安卓可以下载啊？\\n为什么我'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T03:52:48.071427Z",
     "start_time": "2018-08-07T03:52:47.989349Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w2vTXT = s\n",
    "with open('./data/w2vT.txt', 'w') as f:\n",
    "    f.write(w2vTXT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T03:53:20.183824Z",
     "start_time": "2018-08-07T03:52:48.886590Z"
    }
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-a4055886e06d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mw2v\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\" \"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjieba\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcut\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mw2vTXT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/pythonenv/anaconda3/lib/python3.6/site-packages/jieba/__init__.py\u001b[0m in \u001b[0;36m_pcut\u001b[0;34m(sentence, cut_all, HMM)\u001b[0m\n\u001b[1;32m    545\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_lcut_all\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    546\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mHMM\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 547\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_lcut\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    548\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpool\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_lcut_no_hmm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/pythonenv/anaconda3/lib/python3.6/multiprocessing/pool.py\u001b[0m in \u001b[0;36mmap\u001b[0;34m(self, func, iterable, chunksize)\u001b[0m\n\u001b[1;32m    264\u001b[0m         \u001b[0;32min\u001b[0m \u001b[0ma\u001b[0m \u001b[0mlist\u001b[0m \u001b[0mthat\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mreturned\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m         '''\n\u001b[0;32m--> 266\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_map_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmapstar\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstarmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/pythonenv/anaconda3/lib/python3.6/multiprocessing/pool.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    637\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 638\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    639\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mready\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    640\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/pythonenv/anaconda3/lib/python3.6/multiprocessing/pool.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    633\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    634\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 635\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    636\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    637\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/pythonenv/anaconda3/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    549\u001b[0m             \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_flag\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    550\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 551\u001b[0;31m                 \u001b[0msignaled\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    552\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0msignaled\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    553\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/pythonenv/anaconda3/lib/python3.6/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    293\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    294\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 295\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    296\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    297\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "w2v = \" \".join(jieba.cut(w2vTXT))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T03:53:20.185133Z",
     "start_time": "2018-08-07T03:52:50.441Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w2v[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-06T08:39:03.937653Z",
     "start_time": "2018-08-06T08:38:58.676959Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./data/w2v.txt', 'w') as f:\n",
    "    f.writelines(w2v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-06T08:42:26.919583Z",
     "start_time": "2018-08-06T08:40:57.210754Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Word2Vec(LineSentence('./data/w2v.txt'), size=256, window=5, min_count=5, workers=8)\n",
    "model.save('./cache/w2v.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-06T08:43:07.803037Z",
     "start_time": "2018-08-06T08:43:07.762524Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "weights = model.wv.syn0\n",
    "vocab = dict([(k, v.index+1) for k, v in model.wv.vocab.items()])\n",
    "embed_weights = np.zeros(shape=(weights.shape[0]+2, weights.shape[1]))\n",
    "embed_weights[1:weights.shape[0]+1] = weights\n",
    "unk_vec = np.random.random(size=weights.shape[1]) * 0.5\n",
    "embed_weights[weights.shape[0]+1] = unk_vec - unk_vec.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-06T08:49:09.027599Z",
     "start_time": "2018-08-06T08:49:09.025196Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "44409"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-06T08:48:31.871778Z",
     "start_time": "2018-08-06T08:48:31.825985Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pickle.dump(vocab, open('./cache/w2v.dict.pkl', 'wb'))\n",
    "np.save('./cache/word_embed.npy', embed_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-06T08:46:31.404088Z",
     "start_time": "2018-08-06T08:46:31.393963Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('打死', 0.6563826203346252),\n",
       " ('弄死', 0.6158390045166016),\n",
       " ('杀掉', 0.6087832450866699),\n",
       " ('杀人', 0.6080974340438843),\n",
       " ('打爆', 0.5926108360290527),\n",
       " ('杀死', 0.5809358954429626),\n",
       " ('灭', 0.5806133151054382),\n",
       " ('碰见', 0.5758534073829651),\n",
       " ('团灭', 0.5728330016136169),\n",
       " ('灭队', 0.5724929571151733)]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.similar_by_word('杀')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# to one-hot, truncate and padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T07:27:47.155241Z",
     "start_time": "2018-08-07T07:27:47.153416Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sentence_length = 150\n",
    "sen_seq_length = 10\n",
    "word_seq_length = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T07:27:48.403462Z",
     "start_time": "2018-08-07T07:27:48.074020Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./cache/text.csv', 'r') as f:\n",
    "    txt = f.readlines()\n",
    "text = [line.strip() for line in txt]\n",
    "with open('./cache/label.csv', 'r') as f:\n",
    "    label = f.readlines()\n",
    "labels = [int(l.strip())-1 for l in label]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T08:25:11.543958Z",
     "start_time": "2018-08-07T08:25:11.506202Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_embed_dict = pickle.load(open('./cache/w2v.dict.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T03:53:27.320965Z",
     "start_time": "2018-08-07T03:53:27.319309Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_unknow = len(word_embed_dict.keys()) + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T03:59:21.858597Z",
     "start_time": "2018-08-07T03:59:21.855543Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_embed_dict['我']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T03:59:23.206180Z",
     "start_time": "2018-08-07T03:59:23.181119Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:00<00:00, 49642.61it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(100, 150)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = get_sequence(text[:100])\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T03:59:35.119033Z",
     "start_time": "2018-08-07T03:59:35.116223Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   5,  118,  245,   25,   42,    4,    1,  173,    9, 8321, 2515,\n",
       "         35,    5,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0], dtype=int32)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# dataset splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:22:25.184785Z",
     "start_time": "2018-08-07T13:22:20.263094Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('./cache/text.csv', 'r') as f:\n",
    "    txt = f.readlines()\n",
    "text = [line.strip() for line in txt]\n",
    "with open('./cache/label.csv', 'r') as f:\n",
    "    label = f.readlines()\n",
    "labels = [int(l.strip())-1 for l in label]\n",
    "\n",
    "\n",
    "sampleNB = len(labels)\n",
    "shuffle_idx = np.arange(sampleNB)\n",
    "np.random.shuffle(shuffle_idx)\n",
    "text = np.array(text)\n",
    "labels = np.array(labels)\n",
    "text = text[shuffle_idx]\n",
    "labels = labels[shuffle_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:22:25.929632Z",
     "start_time": "2018-08-07T13:22:25.914624Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_test_split = 0.2\n",
    "train_val_split = 0.1\n",
    "\n",
    "train_test_split_idx = int(len(labels)*train_test_split)\n",
    "train_text = text[train_test_split_idx:]\n",
    "train_label = labels[train_test_split_idx:]\n",
    "test_text = text[:train_test_split_idx]\n",
    "test_label = labels[:train_test_split_idx]\n",
    "\n",
    "train_val_split_idx = int(len(labels)*train_val_split)\n",
    "val_text = train_text[:train_val_split_idx]\n",
    "val_label = train_label[:train_val_split_idx]\n",
    "train_text = train_text[train_val_split_idx:]\n",
    "train_label = train_label[train_val_split_idx:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:22:39.301749Z",
     "start_time": "2018-08-07T13:22:39.261742Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('./cache/train_label.npy', train_label)\n",
    "np.save('./cache/test_label.npy', test_label)\n",
    "np.save('./cache/val_label.npy', val_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:25:29.688452Z",
     "start_time": "2018-08-07T13:22:40.694538Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('./cache/train_text.npy', train_text)\n",
    "np.save('./cache/val_text.npy', val_text)\n",
    "np.save('./cache/test_text.npy', test_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# data preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:47:45.898637Z",
     "start_time": "2018-08-07T13:47:41.808193Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_text = np.load('./cache/train_text.npy')\n",
    "train_label = np.load('./cache/train_label.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T12:51:46.133937Z",
     "start_time": "2018-08-07T12:51:46.119502Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(130687,)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_text.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T12:53:09.527028Z",
     "start_time": "2018-08-07T12:53:06.557734Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_text = np.load('./cache/test_text.npy')\n",
    "test_label = np.load('./cache/test_label.npy')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T09:12:52.776224Z",
     "start_time": "2018-08-07T09:12:51.302336Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "session = tf.Session(config=config)\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:47:52.073733Z",
     "start_time": "2018-08-07T13:47:52.032199Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_embed_weight=np.load('./cache/word_embed.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:50:47.274207Z",
     "start_time": "2018-08-07T13:50:47.260123Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_y = np.load('./cache/val_label.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:26:12.746586Z",
     "start_time": "2018-08-07T13:26:12.729609Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_X = np.load('./cache/val_X.npy')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:26:14.515408Z",
     "start_time": "2018-08-07T13:26:14.499468Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 3, 0, 0, 4, 4, 2, 0, 4, 3, 0, 4, 1, 0, 3, 4, 3, 0, 3, 3, 1,\n",
       "       0, 0, 4, 4, 4, 4, 4, 3, 3, 4, 1, 3, 1, 2, 4, 3, 3, 4, 3, 4, 2, 4,\n",
       "       4, 3, 0, 4, 2, 4, 4, 4, 2, 0, 2, 0, 0, 0, 4, 4, 2, 4, 4, 2, 0, 2,\n",
       "       4, 3, 1, 4, 3, 2, 3, 2, 4, 3, 0, 4, 1, 2, 3, 3, 4, 4, 3, 3, 2, 4,\n",
       "       3, 2, 4, 2, 4, 4, 3, 3, 4, 4, 1, 2])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_y[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:26:10.751748Z",
     "start_time": "2018-08-07T13:26:07.273232Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_X = word_text_preprocess(val_text)\n",
    "np.save('./cache/val_X.npy', val_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:50:48.609511Z",
     "start_time": "2018-08-07T13:50:48.589550Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_y = to_categorical(val_y)\n",
    "class_nb = val_y.shape[1]\n",
    "class_nb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TextCNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T12:53:41.964163Z",
     "start_time": "2018-08-07T12:53:17.752445Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 130687/130687 [00:02<00:00, 62787.99it/s]\n"
     ]
    }
   ],
   "source": [
    "# data\n",
    "#test_X = word_text_preprocess(test_text)\n",
    "train_X = word_text_preprocess(train_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T08:57:18.979045Z",
     "start_time": "2018-08-07T08:57:18.952262Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T12:53:49.529353Z",
     "start_time": "2018-08-07T12:53:49.407025Z"
    }
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: './cache/train_X.npy'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-30-59a732fcea79>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mtest_X\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./cache/test_X.npy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mval_X\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./cache/val_X.npy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtrain_X\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./cache/train_X.npy'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/pythonenv/anaconda3/lib/python3.6/site-packages/numpy/lib/npyio.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(file, mmap_mode, allow_pickle, fix_imports, encoding)\u001b[0m\n\u001b[1;32m    370\u001b[0m     \u001b[0mown_fid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbasestring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 372\u001b[0;31m         \u001b[0mfid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"rb\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    373\u001b[0m         \u001b[0mown_fid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    374\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mis_pathlib_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: './cache/train_X.npy'"
     ]
    }
   ],
   "source": [
    "test_X = np.load('./cache/test_X.npy')\n",
    "val_X = np.load('./cache/val_X.npy')\n",
    "train_X = np.load('./cache/train_X.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T08:57:23.813139Z",
     "start_time": "2018-08-07T08:57:23.763007Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('./cache/test_X.npy', test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:26:52.199715Z",
     "start_time": "2018-08-07T13:26:51.337652Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 150)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, 150, 256)     11369216    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, 150, 256)     65792       embedding_1[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 150, 256)     1024        time_distributed_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 150, 256)     0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_1 (Conv1D)               (None, 148, 128)     98432       activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_2 (Conv1D)               (None, 147, 128)     131200      activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_3 (Conv1D)               (None, 146, 128)     163968      activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNor (None, 148, 128)     512         conv1d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 147, 128)     512         conv1d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNor (None, 146, 128)     512         conv1d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 148, 128)     0           batch_normalization_2[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 147, 128)     0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 146, 128)     0           batch_normalization_4[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_1 (GlobalM (None, 128)          0           activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_2 (GlobalM (None, 128)          0           activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d_3 (GlobalM (None, 128)          0           activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv_feat (Concatenate)         (None, 384)          0           global_max_pooling1d_1[0][0]     \n",
      "                                                                 global_max_pooling1d_2[0][0]     \n",
      "                                                                 global_max_pooling1d_3[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 384)          0           conv_feat[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 256)          98560       dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 256)          1024        dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 256)          0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 5)            1285        activation_5[0][0]               \n",
      "==================================================================================================\n",
      "Total params: 11,932,037\n",
      "Trainable params: 561,029\n",
      "Non-trainable params: 11,371,008\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "batch_size=128\n",
    "model_name='textcnn'\n",
    "model=get_textcnn(sentence_length, word_embed_weight, class_nb)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:27:10.928719Z",
     "start_time": "2018-08-07T13:27:10.914713Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_batch_generator = word_text_train_batch_generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:27:51.392807Z",
     "start_time": "2018-08-07T13:27:12.105413Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 38s 37ms/step - loss: 1.1899 - acc: 0.5149 - val_loss: 1.0960 - val_acc: 0.5576\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f21ef92ffd0>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:30:21.928805Z",
     "start_time": "2018-08-07T13:29:47.165238Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 35s 34ms/step - loss: 1.0951 - acc: 0.5570 - val_loss: 1.0726 - val_acc: 0.5690\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f226b1c2eb8>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:31:10.854019Z",
     "start_time": "2018-08-07T13:30:36.085659Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 35s 34ms/step - loss: 1.0610 - acc: 0.5705 - val_loss: 1.1017 - val_acc: 0.5621\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f21ece554a8>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:35:23.303848Z",
     "start_time": "2018-08-07T13:34:13.049967Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 35s 34ms/step - loss: 1.0342 - acc: 0.5829 - val_loss: 1.0507 - val_acc: 0.5799\n",
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 35s 34ms/step - loss: 1.0040 - acc: 0.5949 - val_loss: 1.0561 - val_acc: 0.5816\n"
     ]
    }
   ],
   "source": [
    "for i in range(2):\n",
    "    model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:39:01.660362Z",
     "start_time": "2018-08-07T13:37:51.960840Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 35s 34ms/step - loss: 0.9255 - acc: 0.6284 - val_loss: 1.0447 - val_acc: 0.5862\n",
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 35s 34ms/step - loss: 0.8928 - acc: 0.6410 - val_loss: 1.0539 - val_acc: 0.5837\n"
     ]
    }
   ],
   "source": [
    "K.set_value(model.optimizer.lr, 0.0001)\n",
    "for i in range(2):\n",
    "    model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:50:07.241332Z",
     "start_time": "2018-08-07T13:50:07.226493Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sen_seq_length = 10\n",
    "word_seq_length = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:52:11.628458Z",
     "start_time": "2018-08-07T13:51:58.845199Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_text = np.load('./cache/val_text.npy')\n",
    "val_X = word_han_preprocess(val_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:55:11.022708Z",
     "start_time": "2018-08-07T13:55:09.192700Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_10 (InputLayer)        (None, 10, 15)            0         \n",
      "_________________________________________________________________\n",
      "time_distributed_5 (TimeDist (None, 10, 256)           11665167  \n",
      "_________________________________________________________________\n",
      "bidirectional_10 (Bidirectio (None, 10, 256)           295680    \n",
      "_________________________________________________________________\n",
      "attention_8 (Attention)      (None, 256)               266       \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 256)               1024      \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 5)                 1285      \n",
      "=================================================================\n",
      "Total params: 12,029,214\n",
      "Trainable params: 659,486\n",
      "Non-trainable params: 11,369,728\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "train_batch_generator = word_han_train_batch_generator\n",
    "batch_size=128\n",
    "model_name='han'\n",
    "model=get_han(sen_seq_length, word_seq_length, word_embed_weight, class_nb)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T13:58:25.144804Z",
     "start_time": "2018-08-07T13:55:11.119661Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 97s 95ms/step - loss: 1.1787 - acc: 0.5209 - val_loss: 1.1485 - val_acc: 0.5343\n",
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 95s 94ms/step - loss: 1.0974 - acc: 0.5564 - val_loss: 1.1179 - val_acc: 0.5420\n"
     ]
    }
   ],
   "source": [
    "for i in range(2):\n",
    "    model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T14:01:56.117992Z",
     "start_time": "2018-08-07T13:58:46.003693Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 95s 93ms/step - loss: 1.0523 - acc: 0.5752 - val_loss: 1.0969 - val_acc: 0.5574\n",
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 95s 93ms/step - loss: 1.0082 - acc: 0.5930 - val_loss: 1.0943 - val_acc: 0.5630\n"
     ]
    }
   ],
   "source": [
    "for i in range(2):\n",
    "    model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T14:06:38.612554Z",
     "start_time": "2018-08-07T14:05:03.412927Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 95s 93ms/step - loss: 0.9029 - acc: 0.6376 - val_loss: 1.1130 - val_acc: 0.5639\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f084242a240>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.set_value(model.optimizer.lr, 0.0001)\n",
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-08-07T14:09:15.062437Z",
     "start_time": "2018-08-07T14:07:40.188983Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1020/1020 [==============================] - 95s 93ms/step - loss: 0.8667 - acc: 0.6531 - val_loss: 1.1294 - val_acc: 0.5607\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f082c716e80>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit_generator(train_batch_generator(train_text, train_label, batch_size=batch_size), \n",
    "                    epochs=1, steps_per_epoch=train_text.shape[0]//batch_size, validation_data=(val_X, val_y))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "oldHeight": 399,
   "position": {
    "height": "40px",
    "left": "1414px",
    "right": "20px",
    "top": "94px",
    "width": "453px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "varInspector_section_display": "none",
   "window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}