4.TEXT_CLASSIFICATION/4.2.2.EDA&preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2.2. 데이터 분석 및 전처리"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 데이터 분석하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from wordcloud import WordCloud\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "DATA_IN_PATH = './data_in/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "파일 크기 : \n",
      "ratings.txt                   19.72MB\n",
      "ratings_test.txt              4.94MB\n",
      "ratings_train.txt             14.78MB\n"
     ]
    }
   ],
   "source": [
    "print(\"파일 크기 : \")\n",
    "for file in os.listdir(DATA_IN_PATH):\n",
    "    if 'txt' in file :\n",
    "        print(file.ljust(30) + str(round(os.path.getsize(DATA_IN_PATH + file) / 1000000, 2)) + 'MB')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>document</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9976970</td>\n",
       "      <td>아 더빙.. 진짜 짜증나네요 목소리</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3819312</td>\n",
       "      <td>흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10265843</td>\n",
       "      <td>너무재밓었다그래서보는것을추천한다</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9045019</td>\n",
       "      <td>교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6483659</td>\n",
       "      <td>사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id                                           document  label\n",
       "0   9976970                                아 더빙.. 진짜 짜증나네요 목소리      0\n",
       "1   3819312                  흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나      1\n",
       "2  10265843                                  너무재밓었다그래서보는것을추천한다      0\n",
       "3   9045019                      교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정      0\n",
       "4   6483659  사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...      1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = pd.read_csv(DATA_IN_PATH + 'ratings_train.txt', header = 0, delimiter = '\\t', quoting = 3)\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "전체 학습데이터의 개수: 150000\n"
     ]
    }
   ],
   "source": [
    "print('전체 학습데이터의 개수: {}'.format(len(train_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "train_length = train_data['document'].astype(str).apply(len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    19\n",
       "1    33\n",
       "2    17\n",
       "3    29\n",
       "4    61\n",
       "Name: document, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_length.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\sk8er\\anaconda3\\envs\\pr_tensorflow\\lib\\site-packages\\ipykernel_launcher.py:11: MatplotlibDeprecationWarning: The 'nonposy' parameter of __init__() has been renamed 'nonpositive' since Matplotlib 3.3; support for the old name will be dropped two minor releases later.\n",
      "  # This is added back by InteractiveShellApp.init_path()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Number of review')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtMAAAFNCAYAAADCcOOfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlVUlEQVR4nO3deZhkZX328e/NGgQzKKCRZQQdRIl7WtwVcQN1wBAXCHF7kVEv1zfGiIoRY9wSt5C4TVxwZdUQRlGDyvYqyubGgIQBFwZUcBsRI4j+3j/OaS3aXqpr+nRVTX8/11XXVJ1z6pxfPX2m+66nnvNUqgpJkiRJ87fZsAuQJEmSxpVhWpIkSRqQYVqSJEkakGFakiRJGpBhWpIkSRqQYVqSJEkakGFa0iYvyUOTXDbsOoYpjQ8m+VmS86ZZ/8wk/29ItR2b5J8WaF97Jfl6kuuTvGgh9jnLsV6Z5H1dHkPS6DNMS+pMku8meVTHx9g3yfpplp+Z5NkAVXVOVe3Vx76OTvLRLuocAQ8BHg3sWlX7DKuIRQjtfw+cUVW3rqpjOjwOVfWGqnp2l8eQNPoM05K0CJJsMeQS7gh8t6puGHIdXbsjsLafDdveev8OStoo/hKRtOiSbJ3kHUmuaW/vSLJ1z/q/T/KDdt2zk1SSFRtxvFv0Xid5eZKr26EAlyV5ZJL9gVcCT03yyyTfaLfdOcmpSX6aZF2SI3r2s02SD7VDJy5t6+49znfbY30TuCHJFkmOTHJFe+xLkvxlz/bPTPKlJG9P8vMkVyZ5ULv8qiTXJnnGLK9z2lqTHA68D3hg+9pe20eb3TXJ6e2+LkvylJ51xyZ5Z5JPt6/jq0nu3LP+Me1zNiR5V5Kz2p/j3YD39NTx855D3mam/U1T24FJ1rZtdGa7X5J8EXgE8O/t/u8yzXPPTPL6JF8CfgXcaabXmuT+SX6YZPOe5/9l+/P8o08ykjwgyZfbur6RZN92+SOSfKtnu9OTnN/z+JwkT5zjRyJpRBmmJQ3Dq4AHAPcG7gXsAxwF0IbavwUeBawA9l3IAyfZC3gBcL+qujXwWJoe288CbwBOqKrtqupe7VOOB9YDOwNPAt6QZL923WuA3YE70Qyh+JtpDnko8Hhg+6q6GbgCeCiwDHgt8NEkd+jZ/v7AN4EdgI+3x78fTVv8DU1Q3G6GlzdtrVX1fuC5wLnta3vNHG20LXB6e/zbAYcA70qyd89mh7T13wZYB7y+fe6OwMnAK9rXcBnwIICqunRKHdvPtb9parsLcBzwEmAn4DRgTZKtqmo/4BzgBe3+/2eGl/g0YBVwa+C6mV5rVX0VuAHYr+e5f91uO7WuXYBPA/8E3Bb4O+ATSXYCvgLsmWTHJFsC9wR2TnLrJNsAE23dksaQYVrSMBwG/GNVXVtV19GEqKe1654CfLCq1lbVr4Cj+9jfzm1v4O9vNGOEp/NbYGtg7yRbVtV3q+qK6TZMshvwYODlVfXrqvo6TQ/v03tqfUNV/ayq1gPTjdE9pqquqqr/Baiqk6rqmqr6XVWdAFxO82Zi0neq6oNV9VvgBGA3mra6sar+G7iJJljPt9b5eALNG4wPVtXNVfU14BPAk3u2+c+qOq99g/AxmjdGAI8D1lbVJ9t1xwA/7OOYM+1vqqcCn66q06vqN8BbgG1oA3ufjm3Pr5uB/ed4rcfRvCEiya3b13fcNPv8G+C0qjqt/dmeDlwAPK792Z8PPAz4C+AbwJdofl4PAC6vqp/Mo35JI8QwLWkYdga+1/P4e+2yyXVX9az7/f0ky9uP73+Z5Jc921xTVdv33oBpL3KrqnU0vZpHA9cmOT7JztNt29by06q6fkqtu8xV60zLkjw9zWwTk6H/7sCOPZv8qOf+ZACfumy6num5ap2POwL3n/Lm5DDgz3q26Q3Iv+qp6RZtUlVF01s+l5n2N9Utzp2q+l17vPm8zt6fyVyv9ePAwWmGIR0MXFRV3+OP3RF48jRv6CY/dTiL5lOWh7X3zwQe3t7OmkftkkaMYVrSMFxDEz4mLW+XAfwA2LVn3W6Td6rq++3H99tV1Uxha05V9fGqekhbQwFvnlw1TZ23bXske2u9eq5aew83eSfJHYH/oBlmskMb+i8GMtgrmVet83EVcNaUNyjbVdXz+njuLdokSbhlG01t4/m6xbnT7n835vc6e2uY9bVW1SU04f0AZhji0bOfj0zZz7ZV9aZ2/dQwfRaGaWmTYJiW1LUtk/xJz20Lmo/Jj0qyUzvG9h+AyQu5TgSeleRuSW4FvHohi0kzD/F+bU/jr2l6en/Xrv4RsHvaGR6q6irgy8Ab29rvCRw+pdZXJLlNO2b2BXMcfluaIHddW8uzaHqmN1oftc7Hp4C7JHlaki3b2/0mL/Sbw6eBeyR5Yvuzfj637NH+EbBrkq0GqAuaNn98motGtwReCtxI89oH0c9r/TjwYpogfNIM+/kosDLJY5Ns3v4M9k0y+Ubiy8BeNEN6zquqtbS94sDZA9YuaQQYpiV17TSawDp5O5rmIq0LaC60+xZwUbuMqvoMzTjbM2guRPtKu58bF6ierYE3AT+mGVpwO5qL5eAPQeknSS5q7x9Kc5HhNcB/Aq+pqs+36/6RZgjDd4DP01x4N2OdbS/nW4FzaULlPWjGzi6U2WrtWztU5DE0F+NdQ9NOb6Zpu7me+2Oa8cb/DPwE2JvmZz3ZLl+kmbruh0l+PEBtl9GMT/43mp/hSmBlVd003321++vntR5H04P8xfb1Tbefq4CDaGaEuY6mp/pltH9n2ykJL6IZTz5Z67nA96rq2kFqlzQa0gxnk6TR1PYQXgxs3V4wNrKSPA84pKoePuxaRkXby78eOKyqzhh2PZK00OyZljRy2rl8t05yG5pewjWjGKST3CHJg5Ns1k6591KaHuElrR3qsH07lOaVNGPCvzLH0yRpLBmmJY2i5wDX0szJ/FugnwvfhmEr4L3A9TTDF/4LeNdQKxoND6T52U0Ow3ji5NSAkrSpcZiHJEmSNCB7piVJkqQBGaYlSZKkAW0x7AI2xo477li77777sMuQJEnSJu7CCy/8cVXtNHX5WIfp3XffnQsuuGDYZUiSJGkTl+R70y13mIckSZI0IMO0JEmSNKCxDNNJViZZvWHDhmGXIkmSpCVsLMN0Va2pqlXLli0bdimSJElawsYyTEuSJEmjwDAtSZIkDcgwLUmSJA1oLMO0FyBKkiRpFIxlmPYCREmSJI2CsQzTkiRJ0igY668T1zwdffT8lkuSJGlWhulNnUFZkiSpM4Zp3TJwG74lSZL65phpSZIkaUBj2TOdZCWwcsWKFcMuZdNmj7UkSdKsxrJn2qnxJEmSNArGsmdaQ+BMIJIkSX/EMK2N41AQSZK0hBmmNxWjGGrtzZYkSZs4w7QW3ygGf0mSpAEYpjU6DNmSJGnMGKY1mgzWkiRpDIxlmHae6SXGsdeSJGlEjWWYrqo1wJqJiYkjhl2LRoSBW5IkDcFYhmmpb4ZsSZLUIcO05PhsSZI0IMP0ODP4LTyDtSRJmgfDtDSTmYK1gVuSJLU2G3YBkiRJ0rgyTEuSJEkDcpiHtDEc8iFJ0pJmz7QkSZI0IHumpYWysb3U9nJLkjR2DNNSF2YLxgZlSZI2GWMZppOsBFauWLFi2KVIo8FebUmShmIsw3RVrQHWTExMHDHsWhaUgUiTFmqOa88pSZI6NZZhekkwBGkunheSJA2dYXrcGKAkSZJGhlPjSZIkSQOyZ1paKhw6JEnSgjNMD5uhRpIkaWw5zEOSJEkakD3TXfJjdUmSpE2aPdOSJEnSgOyZHgZ7qSVJkjYJ9kxLkiRJA7JneqHZ6yxJkrRk2DMtSZIkDcieaWkpcqYZSZIWhD3TkiRJ0oBGJkwnuVuS9yQ5Ocnzhl2PJEmSNJdOw3SSDyS5NsnFU5bvn+SyJOuSHAlQVZdW1XOBpwAP7rIuSZIkaSF0PWb6WODfgQ9PLkiyOfBO4NHAeuD8JKdW1SVJDgSeB3yk47oWlmNOJUmSlqROe6ar6mzgp1MW7wOsq6orq+om4HjgoHb7U6vqAOCwLuuSJEmSFsIwZvPYBbiq5/F64P5J9gUOBrYGTpvpyUlWAasAli9f3lmRkiRJ0lxGZmq8qjoTOLOP7VYDqwEmJiaq26qkJcBp8iRp9Pi7eWwMYzaPq4Hdeh7v2i6TJEmSxsowwvT5wJ5J9kiyFXAIcOp8dpBkZZLVGzZs6KRASZIkqR9dT413HHAusFeS9UkOr6qbgRcAnwMuBU6sqrXz2W9VramqVcuWLVv4oiVJkqQ+dTpmuqoOnWH5acxykaEkSZI0DkbmAkRJI8aLXyRJmtPIfJ34fDhmWpIkSaNgLMO0Y6YlSZI0CsYyTEuSJEmjwDHTkubm+GlJkqY1lj3TjpmWJEnSKBjLnumqWgOsmZiYOGLYtUhLjr3UkiT93lj2TEuSJEmjYCx7piWNCHupJUlL3FiG6SQrgZUrVqwYTgGGBml2/fwf8f+RJNkpsQkYy2EezjMtSZKkUTCWYVqSJEkaBWM5zEPSJs6PPSVJY8KeaUmSJGlA9kxLGh/2WEuSRsxYhumhz+YhaeEZjiVJY2gsw7TfgCgtIfOdZs9QLklaRGMZpiVpRgZrSdIiMkxLWhoM2ZKkDhimJW265huaDdySpHkyTEta2gzNkhabv3c2KWMZpp3NQ1LnZuqlnumPoH8cJWlJGssvbamqNVW1atmyZcMuRZIkSUvYWPZMS5IkjRU/vdpkGaYlaaF5IaMkLRmGaUkaBgO3JG0SDNOStBAMxJLAN8pLkGFakobNP76SNLYM05I0SqaG6X6m4jOAS9LQGKYlabF0FXoN1pI0NGMZpv3SFklL0sYEZQO3JHVizjCd5HDg7Kq6fBHq6UtVrQHWTExMHDHsWiRpZPUTmjfm2x0N5ZLUV8/0cuC9SXYHLgTOBs6pqq93WJckaZgMypLUlznDdFW9BiDJNsARwMuAdwCbd1qZJKkbXQdlh5RoqfE8X9L6GeZxFPBgYDvga8DfAed0XJckaalwGImkMdbPMI+DgZuBTwNnAedW1Y2dViVJkjTKfLOnVj/DPO6b5E9peqcfDaxOcm1VPaTz6iRJ48FgIWmJ6meYx92BhwIPByaAq3CYhySpH46flrSJ62eYx5towvMxwPlV9ZtuS5IkbZIM1hp3nreaRj/DPJ7QzuSx3CAtSRo5XsAoaYj6GeaxEngLsBWwR5J7A/9YVQd2XJskSY2pwbiLoGwo1yQ/RdE89DPM42hgH+BMgKr6epI9OqxJkqSNtzHf7ihJfdqsj21+U1UbpiyrLorpV5KVSVZv2DC1LEmSJGnx9NMzvTbJXwObJ9kTeBHw5W7Lml1VrQHWTExMHDHMOiRJHbLXWNIY6CdMvxB4FXAjcBzwOeB1XRYlSVJnNiakL+ZYWsftds821gLoZzaPX9GE6Vd1X44kSWNovuOz+7mvbjh+XgtsxjCd5B1V9ZIka5hmjLSzeUiSlrSF6uGWNNZm65n+SPvvWxajEEmSNIOueq/tFZc22oxhuqoubO/uAHy6qm5cnJIkSRpjXQfUQfY/ijXNZz/z3f9izEsutfq5AHEl8PYkZwMnAJ+tqpu7LUuSJG1SFjPQS4uonwsQn5VkS+AA4FDgnUlOr6pnd16dJElamgbpgZaGoJ+eaarqN0k+Q3Mh4jbAEwHDtCRJ42a+IdULLaVZzRmmkxwAPBXYl+Yrxd8HPKXTqiRJ0txmGxs8ymOjpU1IPz3TT6cZK/0cL0KUJEmzMmRrielnzPShSe4IPBT4fJJtgC2q6vrOq5MkSZs2w7fG3GZzbZDkCOBk4L3tol2BUzqsSZIkSRoL/QzzeD6wD/BVgKq6PMntOq1KkiSNBnuOpVnN2TMN3FhVN00+SLIF03y9uCRJkrTU9BOmz0rySmCbJI8GTgLWdFFMkicm+Y8kJyR5TBfHkCRJkhZKP2H65cB1wLeA5wCnAUf1e4AkH0hybZKLpyzfP8llSdYlORKgqk6pqiOA59JMxydJkiSNrFnHTCfZHFhbVXcF/mPAYxwL/Dvw4Sn7fSfwaGA9cH6SU6vqknaTo9r1kiRJ0siatWe6qn4LXJZk+aAHqKqzgZ9OWbwPsK6qrmzHYx8PHJTGm4HPVNVFgx5TkiRJWgz9zOZxG2BtkvOAGyYXVtWBG3HcXYCreh6vB+4PvBB4FLAsyYqqes/UJyZZBawCWL584IwvSZIkbbR+wvSrO6+iVVXHAMfMsc1qYDXAxMSEs4pIkiRpaPr5BsSzOjju1cBuPY93bZdJkiRJY6Of2Ty6cD6wZ5I9kmwFHAKc2u+Tk6xMsnrDhg2dFShJkiTNpfMwneQ44FxgryTrkxxeVTcDLwA+B1wKnFhVa/vdZ1WtqapVy5Yt66ZoSZIkqQ8zDvNI8oWqemSSN1fVywc9QFUdOsPy02jmrJYkSZLG0mxjpu+Q5EHAgUmOB9K70qnrJEmStNTNFqb/gWYmj12Bt01ZV8B+XRU1lyQrgZUrVqwYVgmSJEnSzGG6qk4GTk7y6qp63SLWNKeqWgOsmZiYOGLYtUiSJGnp6mdqvNclORB4WLvozKr6VLdlSZIkSaNvztk8krwReDFwSXt7cZI3dF2YJEmSNOr6+QbExwP3rqrfAST5EPA14JVdFjYbx0xLkiRpFPQ7z/T2PfeHPrmz80xLkiRpFPTTM/1G4GtJzqCZHu9hwJGdViVJkiSNgX4uQDwuyZnA/dpFL6+qH3ZalSRJkjQG+umZpqp+AJzacS19c8y0JEmSRkG/Y6ZHimOmJUmSNArGMkxLkiRJo2DWMJ1k8yTfXqxiJEmSpHEya5iuqt8ClyVZvkj1SJIkSWOjnwsQbwOsTXIecMPkwqo6sLOq5uAFiJIkSRoF/YTpV3dexTxV1RpgzcTExBHDrkWSJElLVz/zTJ+V5I7AnlX1+SS3AjbvvjRJkiRptM05m0eSI4CTgfe2i3YBTumwJkmSJGks9DM13vOBBwO/AKiqy4HbdVmUJEmSNA76CdM3VtVNkw+SbAFUdyVJkiRJ46GfMH1WklcC2yR5NHASsKbbsiRJkqTR10+YPhK4DvgW8BzgNOCoLouaS5KVSVZv2LBhmGVIkiRpietnNo/fJfkQ8FWa4R2XVdVQh3k4NZ4kSZJGwZxhOsnjgfcAVwAB9kjynKr6TNfFSZIkSaOsny9teSvwiKpaB5DkzsCnAcO0JEmSlrR+xkxfPxmkW1cC13dUjyRJkjQ2ZuyZTnJwe/eCJKcBJ9KMmX4ycP4i1CZJkiSNtNmGeazsuf8j4OHt/euAbTqrSJIkSRoTM4bpqnrWYhYiSZIkjZt+ZvPYA3ghsHvv9lV1YHdlzVnTSmDlihUrhlWCJEmS1NdsHqcA76f51sPfdVpNn5xnWpIkSaOgnzD966o6pvNKJEmSpDHTT5j+1ySvAf4buHFyYVVd1FlVkiRJ0hjoJ0zfA3gasB9/GOZR7WNJkiRpyeonTD8ZuFNV3dR1MZIkSdI46SdMXwxsD1zbbSmSJEkaa0cfPf39TVg/YXp74NtJzueWY6aHNjWeJEmSNAr6CdOv6bwKSZIkaQzNGaar6qzFKESSJEkaN/18A+L1NLN3AGwFbAncUFV/2mVhkiRJ0qjrp2f61pP3kwQ4CHhAl0VJkiRJ42Cz+WxcjVOAx3ZTTn+SrEyyesOGDcMsQ5IkSUtcP8M8Du55uBkwAfy6s4r6UFVrgDUTExNHDLMOSZIkLW39zOaxsuf+zcB3aYZ6SJIkSUtaP2Omn7UYhUiSJEnjZsYwneQfZnleVdXrOqhHkiRJGhuz9UzfMM2ybYHDgR0Aw7QkSZKWtBnDdFW9dfJ+klsDLwaeBRwPvHWm50mSJElLxaxjppPcFvhb4DDgQ8B9q+pni1GYJEmSNOpmGzP9L8DBwGrgHlX1y0WrSpIkSRoDs31py0uBnYGjgGuS/KK9XZ/kF4tTniRJkjS6ZhszPa9vR5QkSZKWGgOzJEmSNCDDtCRJkjSgfr5OXJIkSZre0UcPu4KhsmdakiRJGpA905IkSUvB1B7kJd6jvFDsmZYkSZIGNDJhOsmdkrw/ycnDrkWSJEnqR6dhOskHklyb5OIpy/dPclmSdUmOBKiqK6vq8C7rkSRJ0jwcffQfbppW1z3TxwL79y5IsjnwTuAAYG/g0CR7d1yHJEmStOA6DdNVdTbw0ymL9wHWtT3RNwHHAwd1WYckSZLUhWHM5rELcFXP4/XA/ZPsALweuE+SV1TVG6d7cpJVwCqA5cuXd12rJEmS4JZDPRz28XsjMzVeVf0EeG4f260GVgNMTExU13VJkiRJMxnGbB5XA7v1PN61XSZJkiSNlWGE6fOBPZPskWQr4BDg1PnsIMnKJKs3bNjQSYGSJElSP7qeGu844FxgryTrkxxeVTcDLwA+B1wKnFhVa+ez36paU1Wrli1btvBFS5IkSX3qdMx0VR06w/LTgNO6PLYkSZLUtZH5BsT5cJiHJEmSRsFYhmmHeUiSJGkUjGWYliRJkkaBYVqSJEka0Mh8act8JFkJrFyxYsWwS5EkSRpdflNh58ayZ9ox05IkSRoFYxmmJUmSpFFgmJYkSZIGZJiWJEmSBuQFiJIkSaOs9yLChbygsKv9LjFj2TPtBYiSJEkaBWMZpiVJkqRRYJiWJEmSBmSYliRJkgbkBYiSJEn6Ay9GnJex7Jn2AkRJkiSNgrEM05IkSdIoMExLkiRJAzJMS5IkSQMyTEuSJEkDMkxLkiRJA3JqPEmSpHHUO4XdTPfVubHsmXZqPEmSJI2CsQzTkiRJ0igwTEuSJEkDMkxLkiRJAzJMS5IkSQMyTEuSJEkDMkxLkiRJA3KeaUmSJI2uEZ9Deyx7pp1nWpIkSaNgLMO0JEmSNAoM05IkSdKADNOSJEnSgAzTkiRJ0oAM05IkSdKADNOSJEnSgAzTkiRJ0oAM05IkSdKADNOSJEnSgAzTkiRJ0oDGMkwnWZlk9YYNG4ZdiiRJkpawsQzTVbWmqlYtW7Zs2KVIkiRpCRvLMC1JkiSNAsO0JEmSNCDDtCRJkjQgw7QkSZI0IMO0JEmSNCDDtCRJkjQgw7QkSZI0IMO0JEmSNCDDtCRJkjQgw7QkSZI0IMO0JEmSNCDDtCRJkjQgw7QkSZI0IMO0JEmSNCDDtCRJkjSgLYZdwKQk2wLvAm4Czqyqjw25JEmSJGlWnfZMJ/lAkmuTXDxl+f5JLkuyLsmR7eKDgZOr6gjgwC7rkiRJkhZC18M8jgX2712QZHPgncABwN7AoUn2BnYFrmo3+23HdUmSJEkbrdNhHlV1dpLdpyzeB1hXVVcCJDkeOAhYTxOov84sIT/JKmAVwPLlyxe+aEmSpFF19NHDrqB/vbWOU93zNIwLEHfhDz3Q0IToXYBPAn+V5N3AmpmeXFWrq2qiqiZ22mmnbiuVJEmSZjEyFyBW1Q3As4ZdhyRJktSvYfRMXw3s1vN413aZJEmSNFaGEabPB/ZMskeSrYBDgFPns4MkK5Os3rBhQycFSpIkSf3oemq844Bzgb2SrE9yeFXdDLwA+BxwKXBiVa2dz36rak1VrVq2bNnCFy1JkiT1qevZPA6dYflpwGldHluSJEnq2lh+nbjDPCRJkjQKxjJMO8xDkiRJo2Asw7QkSZI0CgzTkiRJ0oDGMkw7ZlqSJEmjYCzDtGOmJUmSNApSVcOuYWBJrgO+t0iH2xH48SIda1Nhmw3Gdps/22wwttv82WaDsd3mzzYbTJftdseq2mnqwrEO04spyQVVNTHsOsaJbTYY223+bLPB2G7zZ5sNxnabP9tsMMNot7Ec5iFJkiSNAsO0JEmSNCDDdP9WD7uAMWSbDcZ2mz/bbDC22/zZZoOx3ebPNhvMorebY6YlSZKkAdkzLUmSJA3IMD2HJPsnuSzJuiRHDrueUZVktyRnJLkkydokL26X3zbJ6Ukub/+9zbBrHTVJNk/ytSSfah/vkeSr7Tl3QpKthl3jqEmyfZKTk3w7yaVJHui5Nrsk/7f9v3lxkuOS/Inn2h9L8oEk1ya5uGfZtOdWGse07ffNJPcdXuXDM0Ob/Uv7//ObSf4zyfY9617RttllSR47lKJHwHTt1rPupUkqyY7tY8+11kztluSF7Tm3Nsk/9yzv/HwzTM8iyebAO4EDgL2BQ5PsPdyqRtbNwEuram/gAcDz27Y6EvhCVe0JfKF9rFt6MXBpz+M3A2+vqhXAz4DDh1LVaPtX4LNVdVfgXjTt57k2gyS7AC8CJqrq7sDmwCF4rk3nWGD/KctmOrcOAPZsb6uAdy9SjaPmWP64zU4H7l5V9wT+B3gFQPt34RDgz9vnvKv9W7sUHcsftxtJdgMeA3y/Z7Hn2h8cy5R2S/II4CDgXlX158Bb2uWLcr4Zpme3D7Cuqq6sqpuA42l+WJqiqn5QVRe196+nCTe70LTXh9rNPgQ8cSgFjqgkuwKPB97XPg6wH3Byu4ltNkWSZcDDgPcDVNVNVfVzPNfmsgWwTZItgFsBP8Bz7Y9U1dnAT6csnuncOgj4cDW+Amyf5A6LUugIma7Nquq/q+rm9uFXgF3b+wcBx1fVjVX1HWAdzd/aJWeGcw3g7cDfA70XtXmutWZot+cBb6qqG9ttrm2XL8r5Zpie3S7AVT2P17fLNIskuwP3Ab4K3L6qftCu+iFw+2HVNaLeQfNL83ft4x2An/f8EfKc+2N7ANcBH2yHx7wvybZ4rs2oqq6m6an5Pk2I3gBciOdav2Y6t/wb0Z//A3ymvW+bzSLJQcDVVfWNKatst9ndBXhoO2ztrCT3a5cvSrsZprWgkmwHfAJ4SVX9onddNVPHOH1MK8kTgGur6sJh1zJmtgDuC7y7qu4D3MCUIR2ea7fUjvE9iOaNyM7Atkzz8bLm5rk1P0leRTMM8GPDrmXUJbkV8ErgH4ZdyxjaArgtzTDTlwEntp/0LgrD9OyuBnbrebxru0zTSLIlTZD+WFV9sl38o8mPotp/r53p+UvQg4EDk3yXZgjRfjRjgbdvP4oHz7nprAfWV9VX28cn04Rrz7WZPQr4TlVdV1W/AT5Jc/55rvVnpnPLvxGzSPJM4AnAYfWHeXhts5ndmeYN7zfavwu7Ahcl+TNst7msBz7ZDoM5j+bT3h1ZpHYzTM/ufGDP9or3rWgGsZ865JpGUvsO8P3ApVX1tp5VpwLPaO8/A/ivxa5tVFXVK6pq16ranebc+mJVHQacATyp3cw2m6KqfghclWSvdtEjgUvwXJvN94EHJLlV+391ss081/oz07l1KvD0dqaFBwAbeoaDLGlJ9qcZwnZgVf2qZ9WpwCFJtk6yB80FdecNo8ZRU1XfqqrbVdXu7d+F9cB92995nmuzOwV4BECSuwBbAT9msc63qvI2yw14HM2VyFcArxp2PaN6Ax5C89HnN4Gvt7fH0YwB/gJwOfB54LbDrnUUb8C+wKfa+3dq/7OvA04Cth52faN2A+4NXNCeb6cAt/Fcm7PNXgt8G7gY+AiwtefatO10HM248t/QhJnDZzq3gNDM+HQF8C2a2VKG/hpGpM3W0YxVnfx78J6e7V/VttllwAHDrn+U2m3K+u8CO7b3PddmaTea8PzR9vfbRcB+Pdt3fr75DYiSJEnSgBzmIUmSJA3IMC1JkiQNyDAtSZIkDcgwLUmSJA3IMC1JkiQNyDAtSQsoyS873v9L2m9K2+jjtXOvfj7J15M8dWEq/P2+35dk74XcpySNoi3m3kSSNEJeQjOf6q/m2K4f9wGoqnvPtlGSLarq5vnsuKqevRF1SdLYsGdakjqW5M5JPpvkwiTnJLlru/zYJMck+XKSK5M8qV2+WZJ3Jfl2ktOTnJbkSUleBOwMnJHkjJ79vz7JN5J8Jcntpzn+bZOckuSb7Tb3THI7mlB+v7Zn+s5TnnNmknckuQB4cZK/SHJW+xo+l+QOSe6a5Lye5+ye5Fs9z59o7z8myblJLkpyUpLtktwvySfb9Qcl+d8kWyX5kyRXLvCPQJI6Y5iWpO6tBl5YVX8B/B3wrp51d6D5BtEnAG9qlx0M7A7sDTwNeCBAVR0DXAM8oqoe0W67LfCVqroXcDZwxDTHfy3wtaq6J/BK4MNVdS3wbOCcqrp3VV0xzfO2qqoJ4Bjg34Anta/hA8Drq+rbwFbt1/QCPBU4oXcHSXYEjgIeVVX3pfnmyr8FvkbzTZYAD6X55rL7AfcHvjpNLZI0khzmIUkdSrId8CDgpCSTi7fu2eSUqvodcElPr/JDgJPa5T/s7YWexk3Ap9r7FwKPnmabhwB/BVBVX0yyQ5I/7aP8yWC8F3B34PT2NWxO83W+ACfShOg3tf9OHXv9AJo3BV9qn7sVcG5V3ZzkiiR3A/YB3gY8rN33OX3UJkkjwTAtSd3aDPj5LOOSb+y5nxm2mc1vqqra+79lYX+v39D+G2BtVT1wmm1OoHmj8EmgquryKesDnF5Vh07z3LOBA4DfAJ8HjqUJ0y9bgNolaVE4zEOSOlRVvwC+k+TJAGnca46nfQn4q3bs9O2BfXvWXQ/cep5lnAMc1h5/X+DHbV39ugzYKckD231smeTPAdrhIb8FXs2UIR6trwAPTrKife62Se7SU9dLaHqqrwN2oOkFv3g+L06ShsmeaUlaWLdKsr7n8dtoguy7kxwFbAkcD3xjln18AngkcAlwFXARsKFdtxr4bJJresZNz+Vo4ANJvkkzC8gz+nweAFV1U3tx5DFJltH87XgHsLbd5ATgX4A9pnnudUmeCRyXZHJ4y1HA/9CMjb49TQ81wDeBP+vpaZekkRd/Z0nS6EmyXVX9MskOwHnAg6vqh8OuS5J0S/ZMS9Jo+lSS7Wku2HudQVqSRpM905IkSdKAvABRkiRJGpBhWpIkSRqQYVqSJEkakGFakiRJGpBhWpIkSRqQYVqSJEka0P8HxMjeM2UGZIoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 그래프에 대한 이미지 사이즈 선언\n",
    "# figsize: (가로, 세로) 형태의 튜플로 입력\n",
    "plt.figure(figsize=(12, 5))\n",
    "# 히스토그램 선언\n",
    "# bins: 히스토그램 값들에 대한 버켓 범위\n",
    "# range: x축 값의 범위\n",
    "# alpha: 그래프 색상 투명도\n",
    "# color: 그래프 색상\n",
    "# label: 그래프에 대한 라벨\n",
    "plt.hist(train_length, bins=200, alpha=0.5, color= 'r', label='word')\n",
    "plt.yscale('log', nonposy='clip')\n",
    "# 그래프 제목\n",
    "plt.title('Log-Histogram of length of review')\n",
    "# 그래프 x 축 라벨\n",
    "plt.xlabel('Length of review')\n",
    "# 그래프 y 축 라벨\n",
    "plt.ylabel('Number of review')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "리뷰 길이 최대 값: 158\n",
      "리뷰 길이 최소 값: 1\n",
      "리뷰 길이 평균 값: 35.24\n",
      "리뷰 길이 표준편차: 29.58\n",
      "리뷰 길이 중간 값: 27.0\n",
      "리뷰 길이 제 1 사분위: 16.0\n",
      "리뷰 길이 제 3 사분위: 42.0\n"
     ]
    }
   ],
   "source": [
    "print('리뷰 길이 최대 값: {}'.format(np.max(train_length)))\n",
    "print('리뷰 길이 최소 값: {}'.format(np.min(train_length)))\n",
    "print('리뷰 길이 평균 값: {:.2f}'.format(np.mean(train_length)))\n",
    "print('리뷰 길이 표준편차: {:.2f}'.format(np.std(train_length)))\n",
    "print('리뷰 길이 중간 값: {}'.format(np.median(train_length)))\n",
    "# 사분위의 대한 경우는 0~100 스케일로 되어있음\n",
    "print('리뷰 길이 제 1 사분위: {}'.format(np.percentile(train_length, 25)))\n",
    "print('리뷰 길이 제 3 사분위: {}'.format(np.percentile(train_length, 75)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'whiskers': [<matplotlib.lines.Line2D at 0x1866fd5e278>,\n",
       "  <matplotlib.lines.Line2D at 0x1866fd5e518>],\n",
       " 'caps': [<matplotlib.lines.Line2D at 0x1866fd5e828>,\n",
       "  <matplotlib.lines.Line2D at 0x1866fd5eb00>],\n",
       " 'boxes': [<matplotlib.lines.Line2D at 0x1866f9e4e10>],\n",
       " 'medians': [<matplotlib.lines.Line2D at 0x1866fd5edd8>],\n",
       " 'fliers': [<matplotlib.lines.Line2D at 0x1866fd6a2e8>],\n",
       " 'means': [<matplotlib.lines.Line2D at 0x1866fd6a0f0>]}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsYAAAEvCAYAAABGywdiAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAUNklEQVR4nO3df+ztd13Y8eerveLGmELtlWELK87qAs2McMdwsAVlXXEzlD+cgblZtUlTYP6OSmUJ7g+CbotOttGmClINFhGdJQsDOgQZm8BukR8t4LxBkFagl/BDEcMofe+Pe8qu5WLL93y//d7v5fFIbs4578/nnPO6SXPzzKfvc86stQIAgC91Z+33AAAAcDoQxgAAkDAGAIBKGAMAQCWMAQCgEsYAAFDVof0eoOrcc89dF1xwwX6PAQDAGe6mm276yFrr8KmOnRZhfMEFF3T06NH9HgMAgDPczLz/Cx2zlQIAABLGAABQCWMAAKjuRRjPzItm5vaZuflu698/M++ZmVtm5t+etH7VzBybmd+fmUv2YmgAANht9+bDdy+u/lP1y3ctzMy3VJdW37jW+vTMfPVm/RHVU6tHVl9T/feZ+fq11md3e3AAANhN93jFeK31huqjd1t+evXTa61Pb865fbN+afXStdan11p/WB2rHrOL8wIAwJ7Y6R7jr6/+wcy8eWZ+Z2b+7mb9vOoDJ51362YNAABOazsN40PVOdVjqx+rXjYz88W8wMxcMTNHZ+bo8ePHdzgGwJeeSy65pLPOOquZ6ayzzuqSS3ycA2A37DSMb61+c53wlurO6tzqtuqhJ513/mbt86y1rl1rHVlrHTl8+JQ/PgLA3VxyySW95jWv6corr+zjH/94V155Za95zWvEMcAu2Okv3/1W9S3V62bm66v7VR+pXlH96sz8bCc+fHdh9ZZdmBOA6sYbb+zpT396L3jBC6o+d3vNNdfs51gAZ4R783Vt11e/W33DzNw6M5dXL6q+dvMVbi+tLttcPb6leln1rupV1TN9IwXA7llr9bznPe8vrD3vec9rrbVPEwGcOe7xivFa62lf4NC/+ALnP7d67jZDAXBqM9NVV131uSvFVVdddVVf5Mc8ADgFv3wHcIBcfPHFXX311T3jGc/oE5/4RM94xjO6+uqru/jii/d7NIADb06H//125MiRdfTo0f0eA+BAuOSSS7rxxhtbazUzXXzxxb361a/e77EADoSZuWmtdeRUx3b64TsA9okIBtgbtlIAHDDXX399F110UWeffXYXXXRR119//X6PBHBGcMUY4AC5/vrre/azn90LX/jCHv/4x/fGN76xyy+/vKqnPe0LfVYagHvDHmOAA+Siiy7qlltu+bz1Rz7ykd188837MBHAwfKX7TG2lQLgADlVFP9l6wDce8IY4AA6++yze/3rX9/ZZ5+936MAnDHsMQY4gD772c/2hCc8Yb/HADijuGIMcADd9Ut3fvEOYPcIY4AD6K4PTp8OH6AGOFMIYwAASBgDAEAljAEAoBLGAABQCWOAA+uaa67Z7xEAzijCGOCAuvLKK/d7BIAzijAGAICEMcCB9IAHPKCbbrqpBzzgAfs9CsAZw09CAxxAn/zkJ3v0ox+932MAnFFcMQYAgIQxwIF06NChv3ALwPaEMcABdMcdd/yFWwC2d49hPDMvmpnbZ+bmUxz70ZlZM3Pu5vHMzPNn5tjMvGNmHrUXQwMAwG67N1eMX1w96e6LM/PQ6h9Xf3TS8rdVF27+XFFdvf2IAACw9+4xjNdab6g+eopDP1f9eLVOWru0+uV1wpuqB87MQ3ZlUgAA2EM72mM8M5dWt6213n63Q+dVHzjp8a2bNQAAOK190R9nnpn7Vz/ZiW0UOzYzV3Riu0UPe9jDtnkpAADY2k6uGP+t6uHV22fmfdX51Vtn5m9Ut1UPPenc8zdrn2etde1a68ha68jhw4d3MAYAAOyeLzqM11rvXGt99VrrgrXWBZ3YLvGotdaHqldU3735dorHVp9Ya31wd0cGAIDdd2++ru366nerb5iZW2fm8r/k9FdW762OVb9QPWNXpgQAgD12j3uM11pPu4fjF5x0f1XP3H4sAAC4b/nlOwAASBgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVPcijGfmRTNz+8zcfNLav5uZ98zMO2bmv8zMA086dtXMHJuZ35+ZS/ZobgAA2FX35orxi6sn3W3txuqitdbfqf5PdVXVzDyiemr1yM1zXjAzZ+/atAAAsEfuMYzXWm+oPnq3tdeste7YPHxTdf7m/qXVS9dan15r/WF1rHrMLs4LAAB7Yjf2GH9f9d8298+rPnDSsVs3awAAcFrbKoxn5tnVHdVLdvDcK2bm6MwcPX78+DZjAADA1nYcxjPzPdW3V9+11lqb5duqh5502vmbtc+z1rp2rXVkrXXk8OHDOx0DAAB2xY7CeGaeVP149eS11qdOOvSK6qkz8+Uz8/Dqwuot248JAAB769A9nTAz11dPqM6dmVur53TiWyi+vLpxZqretNa6cq11y8y8rHpXJ7ZYPHOt9dm9Gh4AAHbL/P9dEPvnyJEj6+jRo/s9BsBpb3Mx4pROh3/PAU53M3PTWuvIqY755TsAAEgYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAqnsRxjPzopm5fWZuPmntnJm5cWb+YHP7oM36zMzzZ+bYzLxjZh61l8MDAMBuuTdXjF9cPelua8+qXrvWurB67eZx1bdVF27+XFFdvTtjAgDA3rrHMF5rvaH66N2WL62u29y/rnrKSeu/vE54U/XAmXnILs0KAAB7Zqd7jB+81vrg5v6Hqgdv7p9XfeCk827drAEAwGlt6w/frbVWtb7Y583MFTNzdGaOHj9+fNsxAABgKzsN4w/ftUVic3v7Zv226qEnnXf+Zu3zrLWuXWsdWWsdOXz48A7HAACA3bHTMH5Fddnm/mXVDSetf/fm2ykeW33ipC0XAABw2jp0TyfMzPXVE6pzZ+bW6jnVT1cvm5nLq/dX37k5/ZXVP6mOVZ+qvncPZgYAgF13j2G81nraFzj0xFOcu6pnbjsUAADc1/zyHQAAJIwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAqi3DeGZ+eGZumZmbZ+b6mfkrM/PwmXnzzBybmV+bmfvt1rAAALBXdhzGM3Ne9QPVkbXWRdXZ1VOrn6l+bq31ddXHqst3Y1AAANhL226lOFT91Zk5VN2/+mD1rdXLN8evq56y5XsAAMCeO7TTJ661bpuZf1/9UfXn1Wuqm6qPr7Xu2Jx2a3Xe1lMCHDAzc8a851prT14X4HSz4zCemQdVl1YPrz5e/Xr1pC/i+VdUV1Q97GEP2+kYAKelvYzJUwWweAXY3jZbKf5R9YdrreNrrc9Uv1k9rnrgZmtF1fnVbad68lrr2rXWkbXWkcOHD28xBsCXlrXW50L45PsAbGebMP6j6rEzc/85cfniidW7qtdV37E557Lqhu1GBACAvbfjMF5rvbkTH7J7a/XOzWtdW/1E9SMzc6z6quqFuzAnAADsqR3vMa5aaz2nes7dlt9bPWab1wUAgPuaX74DAICEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoNoyjGfmgTPz8pl5z8y8e2a+eWbOmZkbZ+YPNrcP2q1hAQBgr2x7xfjnq1ettf529Y3Vu6tnVa9da11YvXbzGAAATms7DuOZ+crqH1YvrFpr/d+11serS6vrNqddVz1luxEBAGDvbXPF+OHV8eqXZub3ZuYXZ+avVQ9ea31wc86HqgdvOyQAAOy1bcL4UPWo6uq11jdVf9bdtk2stVa1TvXkmbliZo7OzNHjx49vMQYAAGxvmzC+tbp1rfXmzeOXdyKUPzwzD6na3N5+qievta5dax1Zax05fPjwFmMAAMD2dhzGa60PVR+YmW/YLD2xelf1iuqyzdpl1Q1bTQgAAPeBQ1s+//url8zM/ar3Vt/bidh+2cxcXr2/+s4t3wMAAPbcVmG81npbdeQUh564zesCAMB9zS/fAQBAwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCg2oUwnpmzZ+b3Zua/bh4/fGbePDPHZubXZuZ+248JAAB7azeuGP9g9e6THv9M9XNrra+rPlZdvgvvAbBrzjnnnGbmwP+p9n2G3fhzzjnn7PN/EQAnHNrmyTNzfvVPq+dWPzIn/qX+1uqfb065rvqp6upt3gdgN33sYx9rrbXfY7BxV+QD7Ldtrxj/h+rHqzs3j7+q+vha647N41ur87Z8DwAA2HM7DuOZ+fbq9rXWTTt8/hUzc3Rmjh4/fnynYwAAwK7Y5orx46onz8z7qpd2YgvFz1cPnJm7tmicX912qievta5dax1Zax05fPjwFmMAAMD2dhzGa62r1lrnr7UuqJ5a/fZa67uq11XfsTntsuqGracEAIA9thffY/wTnfgg3rFO7Dl+4R68BwAA7KqtvpXiLmut11ev39x/b/WY3XhdAAC4r/jlO4AD6Pinjvc9r/qePvLnH9nvUQDOGMIY4AC65h3X9NYPv7Vr3n7Nfo8CcMYQxgAHzPFPHe+GYze0Wv3Wsd9y1RhglwhjgAPmmndc053rxO8q3bnudNUYYJcIY4AD5K6rxZ+58zNVfebOz7hqDLBLhDHAAXLy1eK7uGoMsDuEMcAB8vbb3/65q8V3+cydn+ltt79tfwYCOIPsyvcYA3DfePmTX77fIwCcsVwxBgCAhDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUPmBD+BL0HrOV9RPfeV+j8HGes5X7PcIAJUwBr4Ezb/5k9Za+z0GGzPT+qn9ngLAVgoAAKiEMQAAVMIYAAAqYQwAAJUwBgCAaoswnpmHzszrZuZdM3PLzPzgZv2cmblxZv5gc/ug3RsXAAD2xjZXjO+ofnSt9YjqsdUzZ+YR1bOq1661Lqxeu3kMAACntR2H8Vrrg2utt27u/2n17uq86tLqus1p11VP2XJGAADYc7uyx3hmLqi+qXpz9eC11gc3hz5UPXg33gMAAPbS1mE8Mw+ofqP6obXWn5x8bJ34aalT/rzUzFwxM0dn5ujx48e3HQMAALayVRjPzJd1Iopfstb6zc3yh2fmIZvjD6luP9Vz11rXrrWOrLWOHD58eJsxAABga9t8K8VUL6zevdb62ZMOvaK6bHP/suqGnY8HAAD3jUNbPPdx1b+s3jkzb9us/WT109XLZuby6v3Vd241IQAA3Ad2HMZrrTdW8wUOP3GnrwsAAPvBL98BAEDbbaUAOLBOfEyC08GDHuQHUoHTgzAGvuSc+CbJg29mzpi/C8DpwFYKAABIGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCg2sMwnpknzczvz8yxmXnWXr0PAADshj0J45k5u/rP1bdVj6ieNjOP2Iv3AgCA3XBoj173MdWxtdZ7q2bmpdWl1bv26P0ATiszc8a8z1prz98D4HSwV2F8XvWBkx7fWv29PXovgNOOmAQ4ePbtw3czc8XMHJ2Zo8ePH9+vMQAAoNq7ML6teuhJj8/frH3OWuvatdaRtdaRw4cP79EYAABw7+xVGP/v6sKZefjM3K96avWKPXovAADY2p7sMV5r3TEz/6p6dXV29aK11i178V4AALAb9urDd621Xlm9cq9eHwAAdpNfvgMAgIQxAABUwhgAACphDAAAlTAGAIBKGAMAQFWz1trvGZqZ49X793sOgAPm3Ooj+z0EwAHzN9dap/zZ5dMijAH44s3M0bXWkf2eA+BMYSsFAAAkjAEAoBLGAAfZtfs9AMCZxB5jAADIFWMAAKiEMcAZa2Z+aGbuv99zABwUtlIAnKFm5n3VkbWW7zoGuBdcMQbYRzPz3TPzjpl5+8z8ysxcMDO/vVl77cw8bHPei2fmO0563ic3t0+YmdfPzMtn5j0z85I54Qeqr6leNzOvm5mzN69x88y8c2Z+eH/+xgCnr0P7PQDAl6qZeWT1r6u/v9b6yMycU11XXbfWum5mvq96fvWUe3ipb6oeWf1x9T+rx621nj8zP1J9y+a1H12dt9a6aPPeD9yTvxTAAeaKMcD++dbq1+/a6rDW+mj1zdWvbo7/SvX4e/E6b1lr3brWurN6W3XBKc55b/W1M/MfZ+ZJ1Z9sOTvAGUcYAxwMd7T5N3tmzqrud9KxT590/7Od4v8GrrU+Vn1j9frqyuoX92pQgINKGAPsn9+u/tnMfFXVZivF/6qeujn+XdX/2Nx/X/Xozf0nV192L17/T6u/vnntc6uz1lq/0YntG4/ahfkBzij2GAPsk7XWLTPz3Op3Zuaz1e9V31/90sz8WHW8+t7N6b9Q3TAzb69eVf3ZvXiLa6tXzcwfVz+0ed27LohctXt/E4Azg69rAwCAbKUAAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAVf0/fYyNokUDM9cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(12, 5))\n",
    "# 박스플롯 생성\n",
    "# 첫번째 파라메터: 여러 분포에 대한 데이터 리스트를 입력\n",
    "# labels: 입력한 데이터에 대한 라벨\n",
    "# showmeans: 평균값을 마크함\n",
    "\n",
    "plt.boxplot(train_length,\n",
    "             labels=['counts'],\n",
    "             showmeans=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "train_review = [review for review in train_data['document'] if type(review) is str]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "wordclud = WordCloud(font_path = DATA_IN_PATH + 'NanumGothic.ttf').generate(' '.join(train_review))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\sk8er\\anaconda3\\envs\\pr_tensorflow\\lib\\site-packages\\seaborn\\_decorators.py:43: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.\n",
      "  FutureWarning\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='label', ylabel='count'>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAADQCAYAAADGUlwqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAARlElEQVR4nO3dfayedX3H8fcHkIkPSJGzTilYszU6ZBPpCVRdzJRYCtssM0p8WjtGrAnoNFu24f5YN5yLZjoHPpA0s9I6p0Ocgxm0a+pTXKxyUAQBCfWBUQK0WqQ+RB3uuz/u3xm35bS9ucp1n55z3q/kzrmu7/W7rvt3Jyf55Hr6/VJVSJLUxRGz3QFJ0txliEiSOjNEJEmdGSKSpM4MEUlSZ4aIJKmzo2a7A+N2wgkn1NKlS2e7G5I0Z9xwww3fraqJmbYtuBBZunQpU1NTs90NSZozkty5v21ezpIkdWaISJI6M0QkSZ0ZIpKkzgwRSVJnC+7prEO1/M82z3YXdBi64e/XzHYXpFlhiEjzyH9f+huz3QUdhk7+q5t7O7aXsyRJnRkikqTODBFJUmeGiCSpM0NEktSZISJJ6qy3EEnyjCQ3Dn32JnlTkuOTbE1yR/u7qLVPksuT7EhyU5LTh461trW/I8naofryJDe3fS5Pkr5+jyTp4XoLkaq6vapOq6rTgOXAj4GPA5cA26pqGbCtrQOcAyxrn3XAFQBJjgfWA2cCZwDrp4OntXnt0H6r+vo9kqSHG9flrLOAb1bVncBqYFOrbwLOa8urgc01sB04LslTgLOBrVW1p6ruB7YCq9q2Y6tqe1UVsHnoWJKkMRhXiLwC+HBbXlxV97Tle4HFbflE4K6hfXa22oHqO2eoP0ySdUmmkkzt3r37UH6HJGlI7yGS5GjgJcBH993WziCq7z5U1YaqmqyqyYmJGWd4lCR1MI4zkXOAr1TVfW39vnYpivZ3V6vfDZw0tN+SVjtQfckMdUnSmIwjRF7JQ5eyAK4Fpp+wWgtcM1Rf057SWgE80C57bQFWJlnUbqivBLa0bXuTrGhPZa0ZOpYkaQx6HcU3yeOBFwOvGyq/DbgqyYXAncD5rX4dcC6wg8GTXBcAVNWeJG8Brm/tLq2qPW35IuBK4Bjgk+0jSRqTXkOkqn4EPHmf2vcYPK21b9sCLt7PcTYCG2eoTwGnPiqdlSQ9Yr6xLknqzBCRJHVmiEiSOjNEJEmdGSKSpM4MEUlSZ4aIJKkzQ0SS1JkhIknqzBCRJHVmiEiSOus1RJIcl+TqJN9IcluS5zrHuiTNH32fiVwGfKqqngk8G7gN51iXpHmjtxBJ8iTgBcD7AarqZ1X1fZxjXZLmjT7PRJ4O7AY+kOSrSf6pzS8y9jnWJUn96DNEjgJOB66oqucAP+KhS1fA+OZYT7IuyVSSqd27d/f9dZK0YPQZIjuBnVX1pbZ+NYNQGfsc61W1oaomq2pyYmLikH6UJOkhvYVIVd0L3JXkGa10FnArzrEuSfNGr9PjAm8APpTkaOBbDOZNPwLnWJekeaHvOdZvBCZn2OQc65I0D/jGuiSpM0NEktSZISJJ6swQkSR1ZohIkjozRCRJnRkikqTODBFJUmeGiCSpM0NEktSZISJJ6swQkSR11muIJPlOkpuT3JhkqtWOT7I1yR3t76JWT5LLk+xIclOS04eOs7a1vyPJ2qH68nb8HW3f9Pl7JEm/aBxnIi+sqtOqano030uAbVW1DNjGQ7MdngMsa591wBUwCB1gPXAmcAawfjp4WpvXDu23qv+fI0maNhuXs1YDm9ryJuC8ofrmGtgOHNdmPjwb2FpVe6rqfmArsKptO7aqtrdh5DcPHUuSNAZ9h0gB/5nkhiTrWm1xm5UQ4F5gcVs+EbhraN+drXag+s4Z6g/jHOuS1I++Zzb8raq6O8kvA1uTfGN4Y1VVkuq5D1TVBmADwOTkZO/fJ0kLRa9nIlV1d/u7C/g4g3sa97VLUbS/u1rzu4GThnZf0moHqi+ZoS5JGpPeQiTJ45M8cXoZWAl8HbgWmH7Cai1wTVu+FljTntJaATzQLnttAVYmWdRuqK8EtrRte5OsaE9lrRk6liRpDPq8nLUY+Hh76vYo4F+q6lNJrgeuSnIhcCdwfmt/HXAusAP4MXABQFXtSfIW4PrW7tKq2tOWLwKuBI4BPtk+kqQx6S1EqupbwLNnqH8POGuGegEX7+dYG4GNM9SngFMPubOSpE58Y12S1JkhIknqzBCRJHVmiEiSOjNEJEmdjRQiSbaNUpMkLSwHfMQ3yWOBxwEntBf9podaP5b9jFMlSVo4DvaeyOuANwFPBW7goRDZC7ynv25JkuaCA4ZIVV0GXJbkDVX17jH1SZI0R4z0xnpVvTvJ84Clw/tU1eae+iVJmgNGCpEkHwR+FbgR+HkrT08EJUlaoEYdO2sSOKWNbyVJEjD6eyJfB36lyxckOTLJV5N8oq0/PcmXkuxI8q9Jjm71X2rrO9r2pUPHeHOr357k7KH6qlbbkeSSh325JKlXo4bICcCtSbYkuXb6M+K+bwRuG1p/O/Cuqvo14H7gwla/ELi/1d/V2pHkFOAVwLOAVcD7WjAdCbwXOAc4BXhlaytJGpNRL2f9dZeDJ1kC/A7wVuBP2uRRLwJe1Zpsase+Alg99D1XA+9p7VcDH6mqnwLfTrKDwQyJADvakPMk+Uhre2uXvkqSHrlRn876XMfj/yPw58AT2/qTge9X1YNtfScPvbR4InBX+74HkzzQ2p8IbB865vA+d+1TP3OmTiRZB6wDOPnkkzv+FEnSvkYd9uQHSfa2z0+S/DzJ3oPs87vArqq64VHp6SGoqg1VNVlVkxMTE7PdHUmaN0Y9E5k+k2DoEtOKg+z2fOAlSc4FHstgqJTLgOOSHNXORpYAd7f2dwMnATuTHAU8CfjeUH3a8D77q0uSxuARj+JbA/8OnH2Qdm+uqiVVtZTBjfFPV9Wrgc8AL2vN1gLXtOVr2zpt+6fbI8XXAq9oT289HVgGfJnBnOvL2tNeR7fvGPVmvyTpUTDqy4YvHVo9gsF7Iz/p+J1/AXwkyd8CXwXe3+rvBz7YbpzvYRAKVNUtSa5icMP8QeDiqvp569frgS3AkcDGqrqlY58kSR2M+nTW7w0tPwh8h8ElrZFU1WeBz7blb/HQ01XDbX4CvHw/+7+VwRNe+9avA64btR+SpEfXqPdELui7I5KkuWfUp7OWJPl4kl3t87H2DogkaQEb9cb6BxjctH5q+/xHq0mSFrBRQ2Siqj5QVQ+2z5WAL1xI0gI3aoh8L8lrpsesSvIaBu9wSJIWsFFD5I+A84F7gXsYvMfxhz31SZI0R4z6iO+lwNqquh8gyfHAOxiEiyRpgRr1TOQ3pwMEoKr2AM/pp0uSpLli1BA5Ismi6ZV2JjLqWYwkaZ4aNQjeCXwxyUfb+suZ4Q1ySdLCMuob65uTTDGYUArgpVXl5E+StMCNPIpvVd1aVe9pn4MGSJLHJvlykq8luSXJ37S6c6xL0jzxiIeCfwR+Cryoqp4NnAasSrIC51iXpHmjtxBp8478sK0+pn2KwSWxq1t9E3BeW17d1mnbz9p3jvWq+jYwPcf6GbQ51qvqZ8D0HOuSpDHp80yEdsZwI7AL2Ap8kxHnWAeG51jfdy71Ew9QlySNSa8hUlU/r6rTGExdewbwzD6/b3+SrEsylWRq9+7ds9EFSZqXeg2RaVX1fQbT4j6XNsd62zTTHOuMOMf6geZe3/f7N1TVZFVNTkw4bqQkPVp6C5EkE0mOa8vHAC8GbsM51iVp3ujzrfOnAJvaU1RHAFdV1SeS3IpzrEvSvNBbiFTVTcwwvpZzrEvS/DGWeyKSpPnJEJEkdWaISJI6M0QkSZ0ZIpKkzgwRSVJnhogkqTNDRJLUmSEiSerMEJEkdWaISJI6M0QkSZ31ORT8SUk+k+TWJLckeWOrH59ka5I72t9FrZ4klyfZkeSmJKcPHWtta39HkrVD9eVJbm77XN6m05UkjUmfZyIPAn9aVacAK4CLk5wCXAJsq6plwLa2DnAOg7lClgHrgCtgEDrAeuBMBqP/rp8OntbmtUP7rerx90iS9tFbiFTVPVX1lbb8AwYTUp0IrAY2tWabgPPa8mpgcw1sZzAD4lOAs4GtVbWnqu5nMFf7qrbt2Kra3iav2jx0LEnSGIzlnkiSpQzmFvkSsLiq7mmb7gUWt+UTgbuGdtvZageq75yhPtP3O8e6JPWg9xBJ8gTgY8Cbqmrv8LZ2BlF998E51iWpH72GSJLHMAiQD1XVv7Xyfe1SFO3vrla/GzhpaPclrXag+pIZ6pKkMenz6awwmDf9tqr6h6FN1wLTT1itBa4Zqq9pT2mtAB5ol722ACuTLGo31FcCW9q2vUlWtO9aM3QsSdIY9DbHOvB84A+Am5Pc2Gp/CbwNuCrJhcCdwPlt23XAucAO4MfABQBVtSfJW4DrW7tLq2pPW74IuBI4Bvhk+0iSxqS3EKmqLwD7e2/jrBnaF3Dxfo61Edg4Q30KOPUQuilJOgS+sS5J6swQkSR1ZohIkjozRCRJnRkikqTODBFJUmeGiCSpM0NEktSZISJJ6swQkSR1ZohIkjozRCRJnfU5FPzGJLuSfH2odnySrUnuaH8XtXqSXJ5kR5Kbkpw+tM/a1v6OJGuH6suT3Nz2ubwNBy9JGqM+z0SuBFbtU7sE2FZVy4BtbR3gHGBZ+6wDroBB6ADrgTOBM4D108HT2rx2aL99v0uS1LPeQqSqPg/s2ae8GtjUljcB5w3VN9fAduC4Nuvh2cDWqtpTVfcDW4FVbduxVbW9DSG/eehYkqQxGfc9kcVtRkKAe4HFbflE4K6hdjtb7UD1nTPUZ5RkXZKpJFO7d+8+tF8gSfp/s3ZjvZ1B1Ji+a0NVTVbV5MTExDi+UpIWhHGHyH3tUhTt765Wvxs4aajdklY7UH3JDHVJ0hiNO0SuBaafsFoLXDNUX9Oe0loBPNAue20BViZZ1G6orwS2tG17k6xoT2WtGTqWJGlMeptjPcmHgd8GTkiyk8FTVm8DrkpyIXAncH5rfh1wLrAD+DFwAUBV7UnyFuD61u7Sqpq+WX8RgyfAjgE+2T6SpDHqLUSq6pX72XTWDG0LuHg/x9kIbJyhPgWceih9lCQdGt9YlyR1ZohIkjozRCRJnRkikqTODBFJUmeGiCSpM0NEktSZISJJ6swQkSR1ZohIkjozRCRJnc35EEmyKsntba71Sw6+hyTp0TKnQyTJkcB7GczRfgrwyiSnzG6vJGnhmNMhApwB7Kiqb1XVz4CPMJivXZI0BnM9RPY3B7skaQx6m0/kcJJkHbCurf4wye2z2Z955ATgu7PdicNB3rH24I00bv5/TlufQz3C0/a3Ya6HyP7mYP8FVbUB2DCuTi0USaaqanK2+yHNxP/P8Zjrl7OuB5YleXqSo4FXMJivXZI0BnP6TKSqHkzyemALcCSwsapumeVuSdKCMadDBKCqrgOum+1+LFBeItThzP/PMUhVzXYfJElz1Fy/JyJJmkWGiDpxuBkdrpJsTLIryddnuy8LgSGiR8zhZnSYuxJYNdudWCgMEXXhcDM6bFXV54E9s92PhcIQURcONyMJMEQkSYfAEFEXIw03I2n+M0TUhcPNSAIMEXVQVQ8C08PN3AZc5XAzOlwk+TDwReAZSXYmuXC2+zSf+ca6JKkzz0QkSZ0ZIpKkzgwRSVJnhogkqTNDRJLUmSEi9SjJDw+yfekjHW02yZVJXnZoPZMeHYaIJKkzQ0QagyRPSLItyVeS3JxkeNTjo5J8KMltSa5O8ri2z/Ikn0tyQ5ItSZ4yS92X9ssQkcbjJ8DvV9XpwAuBdyZJ2/YM4H1V9evAXuCiJI8B3g28rKqWAxuBt85Cv6UDOmq2OyAtEAH+LskLgP9lMHT+4rbtrqr6r7b8z8AfA58CTgW2tqw5ErhnrD2WRmCISOPxamACWF5V/5PkO8Bj27Z9xx4qBqFzS1U9d3xdlB45L2dJ4/EkYFcLkBcCTxvadnKS6bB4FfAF4HZgYrqe5DFJnjXWHksjMESk8fgQMJnkZmAN8I2hbbcDFye5DVgEXNGmHX4Z8PYkXwNuBJ433i5LB+covpKkzjwTkSR1ZohIkjozRCRJnRkikqTODBFJUmeGiCSpM0NEktSZISJJ6uz/ANml+aXJ7A1HAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x216 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axe = plt.subplots(ncols=1)\n",
    "fig.set_size_inches(6, 3)\n",
    "sns.countplot(train_data['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "긍정 리뷰 개수: 74827\n",
      "부정 리뷰 개수: 75173\n"
     ]
    }
   ],
   "source": [
    "print(\"긍정 리뷰 개수: {}\".format(train_data['label'].value_counts()[1]))\n",
    "print(\"부정 리뷰 개수: {}\".format(train_data['label'].value_counts()[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "train_word_counts = train_data['document'].astype(str).apply(lambda x:len(x.split(' ')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\sk8er\\anaconda3\\envs\\pr_tensorflow\\lib\\site-packages\\ipykernel_launcher.py:4: MatplotlibDeprecationWarning: The 'nonposy' parameter of __init__() has been renamed 'nonpositive' since Matplotlib 3.3; support for the old name will be dropped two minor releases later.\n",
      "  after removing the cwd from sys.path.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Number of reviews')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4AAAAJjCAYAAABdk3/PAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA7yUlEQVR4nO3de7xtVV03/s9XQEGlgwKagQYGqWSFeiLNG2UqmMdbpdLj0808WVLWYylmpVZmZl6fvFEiPf1MwDtH8ZaK4DUO5o1ExcQ4plzlqCkgMn5/zLllsdl7n7XPWWuvfdZ8v1+v9dp7jrnWXN811kT354wxx6zWWgAAAJh/N5l1AQAAAKwNARAAAGAgBEAAAICBEAABAAAGQgAEAAAYCAEQAABgIARAgBFV9ayqumzGNbSqOn6J9kP6fQ8dabuwqv5uFcfeXFWPmFCpc6GqblFVp1TV5X3//vqsa1pOVW2tqpNnXcdqVNWP9v9d7TfGc0+uqq1rUNYuW+6/U4D1bs9ZFwDALnlkkstX8fzNST6T5C1TqWb39DtJNiX51SRfSfLF2ZYzd340yTOTnJzkyh089y+T7DPleiblXkm+NOsiAFZLAATYjbXW/n3WNexIVe3TWvvOrOtYwZ2TfK619sZZF5LsFv01Na21qYXvSfdra+2jkzoWwFoyBRRglarq56rqY1V1VVVdXFUvr6pbLnrOT1TVh/vnnFdVD5nG9L3FU0Cr6seq6p1VdUVV/U9VfbaqntTvOzPJPZL8Wj997fvTHatqj36a3n9V1dV9zb+yxPsdX1UX9cd+S1U9oD/O0SPPaVX1f6rqxVV1aZJP9+2/UFXvqapLquobVfXRqnrQouM/q6ouq6qf7vvrO1X1wao6tKpu07/nt/rP9XNj9M8BVfVP/fTOb1fVmVW1cbT/kjw+yd0W+mSZ4/xG/5n3Gmn77/641W/fpKqurKonjDxnxXOlqo7u3/fBVXV6VX0ryd/3++5aVR/qX/vZqnrYjj7vyHGfUFWfHnnfN1TVhpH9j+73X91/n8+pqj1H9i85FXrxtMeF86+q/rCqtlXV16ubTrvfwudLsqV/+pf611+4Qt03mAJaVb/ev+bH+3Pnf6rq/Kp61Bh9sNx5uHdV/W3/ua+uqk9W1UMW1XDOEsd7Un8O7btUX/RtD+/P26uq6mv9++zV7zu0f83PjDz/dX3bT4y0bamq1+7o8wHsLAEQYBWq6seSvDPJZUl+Md3Utl9J8oaR59w8ybvSTWU7LslfJXlRkjus4q1uUlV7jj6S7DHG67Yk+V6SxyV5WJL/m2Tfft/vJjk/yRnppq/dK8nb+31/keQZSU7sX/ehJK+tquNGPtcj++Odnm7q6aeSvHqZOv44ye2S/O8kv9+3HdrX97/T9d2Hk7yjqu696LU37+t4Ubr+u0OSf07yuiQfTPKodFM1X9/39UrekuTBSf4oyWPS/f/e+6vqsH7/I/v+OH+kT5Zydl/X3ZOkqg5Pcpt0fXtE/5yfTLKhf+5Y58qIVyf5ZLq+f3VV7ZPuHLpl/5q/SvLijHEOVdWfJnlVkg8keUS6Ka7b+2OlutB9apKPJ3l4uu/0j9IHz53w6CQPSDe9+GlJHprkr/t9H++PnXTf273S9flq/UuuP+++kOSUqjp4jNctdR6+Icmv9zVuSnJOktOr6sh+/6lJNlbVoYuO9ZgkZ7TWvrnUG1XVo5O8Kcm/pfsen52uT56bJK21L6U7b+878rL7Jrlqoa2qbpLk3unPIYCpaK15eHh4ePSPJM9KctkK+09J9wfoHiNtj07Sktyr335SkmuSHDTynKP655w8Rg1tB4+Hjjz3wiR/1/9+QL//x1c49tbFNSS5dZL/SfLMRe1npJsaubB9TpK3L3rOy/v3PHpR/R/fwWe8SbrLEN6V5KRF/d+S3H+k7Xf7tj8faTuibzt2hfc4Zolj3SLJpUleNdJ2cpKtY3wv/53kj/rffzPJuUk+kuSJfdvvJ7lklefK0f32ixa91+8m+W6Sg0fa7r2jcyjJfkm+neSFKzzno0nev6jtqen+4eDglf476N//+EXn3xeT7DnS9uIkXxvZfmj/ukPG6OMbfBfpglpL8psjbfsnuXah33fw39HHF7U9YPE50befleT1/e97pgvtJ4zsPyjJdUl+aam+SFJJvpzkNYuO+5tJvpNk/377dUne1v9+x77PX57klL7tJ/rj/tiO+srDw8NjZx9GAAFW56gkb26tfW+k7Y3p/iC9T7/9U0nOba19ZeEJrbV/S3LxwnZ1Rkf4Fo/uPb8/zuhjR1MAr0hyUZJXVtVjquo2Y36mu6Yb3Xr9ovZTk/xoVR3Yj0DeLd0ozKjF2wvOWNxQVQdXNx3zK+n667tJHpRukZBR1+SGIyAX9D/ft0TbQcu8f9J9V5e01j6w0NBa+58kb8v139VqnJ3rR2/uly40nLWo7YOL3n9H58qCty/aPirdObRtpPYPJblkBzXeK93I82uW2tmfZ3fP0t/1TbL8COhK3t9au3Zk+z+S3KZGpstOwLsXfmmtXZ6uH8YZAVx8Hv58kq8l+dCi0fX3JtnYH//adCN5jxl53S+n+0eSxd/Tgh9NNzp72qLjvi/J3un+G0u68+Xe/Ujf/dKNom/JDc+hK9L1IcBUCIAAq3O7jAS5JOn/wL883UhakvxgulGmxUbb7p8uAC083rvouf/VWts6+kh/DdNyWmvXpQtUX0tyUpKvVdXZVXW3MT5TFn+uke1bpxtd3CM3/lxLfc4bHav/g/f0JD+T5M+T/Gy6UPuOdH8gj/pm/1kWXNP/vHKhobW20Lb4taNul6UD08W5/rtajbOT3KeqKt0f7GfnhqHwPrlhcB3nXBmtadQPLlP7jgLg/v3Pry6z/4Akey3xfqPf9WpduWj7mnQjYjfbiWOt5j1W+u4XLP6cB6Tr2+8uejwrye1HnndKkiOrauEfJx6T5PS2/CIyB/Q/z1h03IVVQheOfXa6Udq75vpz6MNJfrCq7ti3fbC1tuS1qACTYBVQgNX5arprv76vH1XZP92/3CddALvTEq89cOT3c9MFoAVLXle0Wq2185P8Yj/6ct8kz0vy9qo6eFGoGrUQFm6TG95S4rb9zyuSfD3ddLXRz5Altr9fyqLtw9KNIB7bWnvnQmN/rdu03Oi76t02139Xq3F2uoD0wHTXM56dbjTvoP66utvmhgFwnHNlweL++lq61UkX29Go7sL3d7t00xgXuyxdMFl8nNHvOumuS7vp6BOq6lY7eO/1aHG/XpHuOrxH7OB1H0gXHh9TVf8vyT3TX8u3jIV+25xkqZV5F4Lgef1z75tutO/prbXtVfWpvu2+SV64g9oAdokRQIDV+ViSRy6asvmodP+gtjD975wk96iq709PrKqjcv0f2WmtfXPRCN/nJllka+27rbX3pftj8nbpRh2SpUdOPpPuurFfXtT+6CSfb61d2k+L+/d0i4aMGndlyoWgd/VCQ1X9cLrr2qblY+mmIt5v5D1vnuQXcsOpmuP6dLqRqGckOb/vl6+n679nJPlWbvjH/zjnynIWzqHvT3PsF8vZUQD8SLprzn5tqZ39COS5Wfq7vq5/fZJsS7Lv6DmcbnR5Z4wzWrtW3ptuBPBbi0fY+1H2JN/vp9enG/l7dLrv/Z1LHbD3uXTB8pCljttPW00/svfB/piHpZsSmv7nb6b7b9UCMMBUGQEEuLGbVtUvLdH+gXSrMf57krdU1SvSXYf0vCTvaq0t/PH8miR/muRtVfXsdOHn2emmSy43CrfL+qXk/y7d9Vz/meRW6VZl/GRrbWGE4vwkD66qB6cbLfpSa+3yqnpxkj+tqmvTLRTzqCQPSbcK54LnJnljVf19uumc904XpjLG5zo/Xah4QVX9WbrVM5+d7o/mqWitvauqPpzk1Ko6Id3n/aN038fzd+J411XVh9J95leN7Do73cI/71l0vd8458pyFs6ht1fVs/qa/zJLj+qN1nhlVf1lkudU1U3TTUm8WV/zs/vrUp+Z5F1V9Zp0Ux1/vD/2P4xcc/jOdEHypKp6QboRzyfuoOblLPzjxm9X1SlJvt1aW3E68xS9J93CQ++pquelG5H7gSRHJtm7tfb0keeemuT4JH+Y5C0j045vpD83npLkn6vqB9JNbb4m3UIvj0i3eMy3+6efne78+1xr7ZKRtt9P9w8xH5/A5wRYlhFAgBvbN92//i9+/Fhr7bwkx6YbiXlTuj/yX5fk+4Gx/0PvmHR/QJ+a7vqip6YbRfjGFOv+Wrppa89I9wfoy5N8Njccpfurvu20dKNMm/r2P08X8H4n3SIp90vyuNbaKQsvbK29Kd0fqY9Id3uFn8r1S/yv+Llaa1enC5XXpluG/y/79/vASq+bgEek+6P/xem+w0ryc621C1Z4zUoWRmfOWqLtBqN645wry+nPoQenW3jklHSh7SnpVprc0WsXvsefT/LWdGF1v/TTjFtr707y2HSLnmxJ8gdJXpAu7CwcY+HWFQen+64fl+52FKvWWvtyuvPkUeluL7Jl5VdMTz8C96h018j+Qbow+Kp0i98sHpX9ULpFlW6X7jvY0bFPTTdCfmS6c+1N6VZz/XiuHwVNVj6HPtZa++64nwdgZ5TrjAGmr7+n2OeTbG6tLblC4+6ov+fcM5LceoUFMgCAdcIUUIApqKqnp7tv3JfTLQ//9HRTQN84y7p2RVUdmO5zvD/dVLX7ppti+mrhDwB2DwIgwHS0dNP2fijdwidnp7uJ+DSngE7bNelWpvzVJBvSrXL5kiR/NsuiAIDxmQIKAAAwEBaBAQAAGIi5mwJ6wAEHtEMOOWTWZQAAAMzEueeee1lr7cCl9s1dADzkkEOydevWHT8RAABgDlXVsrcNMgUUAABgIARAAACAgRAAAQAABmLurgEEAACG7bvf/W62bduWq666atalTNXee++dgw8+OHvttdfYrxEAAQCAubJt27bsu+++OeSQQ1JVsy5nKlprufzyy7Nt27YceuihY7/OFFAAAGCuXHXVVdl///3nNvwlSVVl//33X/UopwAIAADMnXkOfwt25jMKgAAAAAMhAAIAAPOtarKPHbjyyivz8pe/fNVlPuQhD8mVV165Ex9wfAIgAADABC0XAK+99toVX3fGGWdkv/32m1JVHauAAgAATNAJJ5yQL37xiznyyCOz1157Ze+9986tbnWrnH/++fn85z+fRzziEbnoooty1VVX5clPfnI2b96cJDnkkEOydevWfOtb38qxxx6b+9znPvnwhz+cgw46KG9961uzzz777HJtRgABAAAm6G/+5m/yIz/yI/nEJz6R5z//+fn4xz+el7zkJfn85z+fJDnppJNy7rnnZuvWrXnpS1+ayy+//EbH+MIXvpAnPelJOe+887LffvvljW9840RqMwIIAAAwRUcdddQN7tX30pe+NG9+85uTJBdddFG+8IUvZP/997/Baw499NAceeSRSZJ73OMeufDCCydSiwAIAAAwRbe4xS2+//uZZ56Zf/3Xf81HPvKR3PzmN8/RRx+95L38bnazm33/9z322CPf+c53JlLL3EwBrapNVXXi9u3bZ10KAAAwYPvuu2+++c1vLrlv+/btudWtbpWb3/zmOf/88/PRj350TWubmxHA1tqWJFs2btz4hFnXAgAArCOtrenb7b///rn3ve+du971rtlnn31y29ve9vv7jjnmmLzyla/MXe5yl9zpTnfKPe95zzWtrdoad8a0bdy4sW3dunXWZQAAADPy2c9+Nne5y11mXcaaWOqzVtW5rbWNSz1/bqaAAgAAsDIBEAAAYCAEQAAAYO7M26VuS9mZzygAAgAAc2XvvffO5ZdfPtchsLWWyy+/PHvvvfeqXjc3q4ACAAAkycEHH5xt27bl0ksvnXUpU7X33nvn4IMPXtVrBEAAAGCu7LXXXjn00ENnXca6JACysqqdf+0cD7kDAMDuyDWAAAAAAyEAAgAADIQACAAAMBCuARyCXbmODwAAmBtGAAEAAAbCCCDrk9VHAQBg4owAAgAADIQACAAAMBACIAAAwEAIgAAAAANhERjmz67e9sIiMgAAzCkjgAAAAAMhAAIAAAyEAAgAADAQAiAAAMBAWARmrezKwiQWJQEAACbACCAAAMBAGAHcHezqbQ1YHaO1AADMKSOAAAAAAyEAAgAADIQACAAAMBACIAAAwECs+wBYVbeoqq1V9dBZ1wIAALA7W/MAWFUnVdUlVfWZRe3HVNXnquqCqjphZNfTkpy2tlUCAADMn1mMAJ6c5JjRhqraI8nLkhyb5Igkx1XVEVX1wCT/keSStS4SAABg3qz5fQBba2dV1SGLmo9KckFr7T+TpKpOSfLwJLdMcot0ofA7VXVGa+26xcesqs1JNifJHe5whylWDwAAsPtaLzeCPyjJRSPb25L8dGvt+CSpql9PctlS4S9JWmsnJjkxSTZu3OhO3AAAAEtYLwFwRa21k2ddA4ylaudf2/zbBQAA07VeVgH9SpLbj2wf3LcBAAAwIeslAJ6T5PCqOrSqbprksUlOn3FNAAAAc2UWt4F4XZKPJLlTVW2rqse31q5NcnySdyX5bJLTWmvnrXVtAAAA82wWq4Aet0z7GUnO2NnjVtWmJJsOO+ywnT0EAADAXFsvU0B3WWttS2tt84YNG2ZdCgAAwLo0NwEQAACAlQmAAAAAAyEAAgAADIQACAAAMBBzEwCralNVnbh9+/ZZlwJrr2rnHwAADMbcBECrgAIAAKxsbgIgAAAAKxMAAQAABkIABAAAGAgBEAAAYCDmJgBaBRQAAGBlcxMArQIKAACwsrkJgAAAAKxMAAQAABgIARAAAGAgBEAAAICBEAABAAAGQgAEAAAYiLkJgO4DCDupatceAADsNuYmALoPIAAAwMrmJgACAACwMgEQAABgIARAAACAgRAAAQAABkIABAAAGAgBEAAAYCAEQAAAgIEQAAEAAAZibgJgVW2qqhO3b98+61IAAADWpbkJgK21La21zRs2bJh1KQAAAOvS3ARAAAAAVrbnrAsAdnNVO//a1iZXBwAAO2QEEAAAYCAEQAAAgIEQAAEAAAZCAAQAABgIARAAAGAgBEAAAICBEAABAAAGYm4CYFVtqqoTt2/fPutSAAAA1qW5CYCttS2ttc0bNmyYdSkAAADr0twEQAAAAFYmAAIAAAyEAAgAADAQAiAAAMBACIAAAAADIQACAAAMhAAIAAAwEAIgAADAQAiAAAAAA7HnrAsABqxq51/b2uTqAAAYCCOAAAAAAyEAAgAADIQACAAAMBBzEwCralNVnbh9+/ZZlwIAALAuzU0AbK1taa1t3rBhw6xLAQAAWJfmJgACAACwMgEQAABgIARAAACAgRAAAQAABkIABAAAGAgBEAAAYCAEQAAAgIEQAAEAAAZiz1kXALDmqnbt9a1Npg4AgDVmBBAAAGAgBEAAAICBEAABAAAGQgAEAAAYCAEQAABgIARAAACAgRAAAQAABkIABAAAGAgBEAAAYCDmJgBW1aaqOnH79u2zLgUAAGBdmpsA2Frb0lrbvGHDhlmXAsy7qp1/AADM0NwEQAAAAFYmAAIAAAyEAAgAADAQAiAAAMBACIAAAAADseesCwAYlF1ZCbS1ydUBAAySEUAAAICBEAABAAAGQgAEAAAYCAEQAABgIARAAACAgRAAAQAABkIABAAAGAj3AQTYXbiHIACwi4wAAgAADIQACAAAMBCmgAKwsl2ZepqYfgoA64gRQAAAgIEQAAEAAAZCAAQAABgIARAAAGAgBEAAAICBsAooANPlBvYAsG4YAQQAABgIARAAAGAgBEAAAICBEAABAAAGwiIwAKxfFpABgIkyAggAADAQAiAAAMBArOspoFV1lyRPTnJAkve21l4x45IA2F2YPgoAN7LmI4BVdVJVXVJVn1nUfkxVfa6qLqiqE5KktfbZ1toTkzw6yb3XulYAAIB5MospoCcnOWa0oar2SPKyJMcmOSLJcVV1RL/vYUnenuSMtS0TAABgvqx5AGytnZXkikXNRyW5oLX2n621a5KckuTh/fNPb60dm+R/LXfMqtpcVVurauull146rdIBAAB2a+vlGsCDklw0sr0tyU9X1dFJHpXkZllhBLC1dmKSE5Nk48aNLtwAAABYwnoJgEtqrZ2Z5MwZlwEAADAX1sttIL6S5PYj2wf3bQAAAEzIegmA5yQ5vKoOraqbJnlsktNnXBMAAMBcmcVtIF6X5CNJ7lRV26rq8a21a5Mcn+RdST6b5LTW2nlrXRsATETVzj8AYIrW/BrA1tpxy7SfkV241UNVbUqy6bDDDtvZQwDA7LmBPQBTtF6mgO6y1tqW1trmDRs2zLoUAJgNI48A7MDcBEAAAABWJgACAAAMhAAIAAAwEAIgAADAQMxNAKyqTVV14vbt22ddCgAAwLo0NwHQKqAAAAArW/P7AAIA65D7DwIMwtyMAAIAALAyARAAAGAgTAEFAFitXZkym5g2C8zM3IwAWgUUAHZTVTv/AGBV5iYAWgUUAABgZXMTAAEAAFiZawABgGEyhRQYIAEQANh9CXEAq2IKKAAAwEAYAQQA2J3syqin20/A4BkBBAAAGIi5CYDuAwgAALCyuQmA7gMIAACwsrkJgAAAAKzMIjAAAOyYxWdgLgiAAABD4b6JMHimgAIAAAyEAAgAADAQAiAAAMBAuAYQAIDp2tVrDy0iAxMz1ghgVe1ZVTdb1PagqvqDqrr7dEpbHTeCBwAAWNm4U0BPTfKKhY2q+v0k70zy3CQfraqHTqG2VXEjeAAAgJWNGwDvmeSMke0/TvKC1to+Sf4xyTMmXRgAAACTNW4A3D/J15Kkqn48yQ8leWW/7/VJjph8aQAAAEzSuAHw4iSH9L8fk+TLrbUv9tv7JLluwnUBAAAwYeOuAvr6JM+rqp9M8htJ/n5k392SfGHShQEAADBZ4wbAE5J8I8lPpVsM5q9H9t0j3SIxAAAArGNjBcDW2rVJ/mKZfY+aaEUAAABMxbj3AfznqnpCVd152gUBAAAwHeNOAd0vyd8k2a+qrkjywSRn94+Pt9a+N53yAAAAmJSxRgBba5uSHJBuwZdnJ7k6yVOSfDTJlVX1nqlVCAAAwESMOwKY1lpL8qkkn6qqtyW5X5Lf7H/+3HTKAwAAYFLGCoBVddck9+0f90ty23Rh8Owk/7f/OVNVtSnJpsMOO2zWpQAAAKxL1Q3s7eBJVdcl+U6S1yTZkuQjrbVvTLm2nbJx48a2devWWZdxY1WzrmDtjXFuLUt/rY7+Wh39tTr6a/WG1mf6a/X8N7k6u3qOwcBU1bmttY1L7Rt3CuipSe6T5IlJ7pXk7Ko6K8lZrbXLJlMmAAAA0zTuIjDHtdZun+TwJC9Ncst0q4JeXFX/UVWvmGKNAAAATMDYi8AkSWvtS0m+VFXnJvl4kkenuybwTkl+Z/LlAQDALtiVKbOmnjKHxl0E5p65fhGYe6e7L+AVST6U5I+zDhaBAQAAYGXjjgB+OMlX0gW9ZyQ5u7V23tSqAgAAYOLGDYB3bK1dOM1CAAAAmK6xAuBC+KuqWyW5a5LbJ3lHa+3rVbV3kmtaa9dNrUoAAAB22VirgFbVnlX1t0m2JflAkn9Ocmi/+41Jnjmd8gAAAJiUsQJgkuckeUKS45PcMcnockpvTbJpwnUBAAAwYeNeA/irSU5orb2mqvZYtO+L6UIhAAAA69i4I4D7pQt6S7lpksWhEAAAgHVm3AD4mSQPX2bfseluCg8AAMA6Nu4U0L9K8saq2ifJ65O0JEdW1SOT/HaSh02pvrFV1aYkmw477LBZlwIAALAujTUC2Fp7a5JfSfLzSd6RbhGYf0zy60n+d2vtXdMqcFyttS2ttc0bNmyYdSkAAADr0rgjgGmtnZbktKr60SQHJLkiyedaa21axQEAwMxU7fg5y/EnMuvU2AFwQWvt80k+P4VaAAAAmKJlA2BV/W6S17fWLu1/X0lrrb1isqUBAAAwSSuNAP59kq1JLu1/X0lLIgACAACsY8sGwNbaTZb6HQAAgN3TWMGuqm4x7UIAAACYrnFH9i6pqlOr6pFVdbOpVgQAAMBUjBsAn5rkB5O8IV0Y/Oeq+oWqWvUqogAAAMzGuDeCf1lr7f5Jbp/kmUl+JMmWJBdX1aur6oFTrBEAAIAJWNXiLq21/26tvbi19jNJDkny3CTHJHnHFGoDAABggnZqCmdVHZbkMf3jdkkummRRAAAATN7YI4BV9cNV9dSqOjfJ55Icn+TMJPdtrf3wlOoDAABgQsYaAayqf0tyjyRXJHlTkj9K8oHW2nVTrA0AAIAJGncK6HlJ/jzJe1pr35tiPQAAAEzJWAGwtfYb0y4EAACA6VrNNYA/0d8M/otVdXVV3b1vf05VHTu9EgEAAJiEsQJgH/DOTXcz+P+XZK+R3Vcn+b3JlwYAAMAkjTsC+NwkJ/c3g3/Oon2fSHLkBGsCAABgCsYNgHdOcmr/e1u07xtJbj2xigAAAJiKcQPgJUnuuMy+H0vyX5MpZ+dV1aaqOnH79u2zLgUAAGBdGjcAnpLkL6rqPiNtrap+NMnTkrx24pWtUmttS2tt84YNG2ZdCgAAwLo07n0A/yzJEUk+kORrfdtb0y0K8+4kfz350gAAAJikce8DeHWSh1bVA5I8IMkBSa5I8t7W2numWB8AAAATssMAWFV7J/lUkt9vrb0zyXunXhUAAAATt8NrAFtrVyXZL8l1U68GAACAqRl3EZjXJvmNaRYCAABzo2rnHzBF4y4C819JHl1V5yR5R5KLc8P7AbbW2ismXRwAAACTM24AfEH/83ZJ7rHE/pZEAAQAAFjHxl0FdNypogAAAKxTgh0AAMBACIAAAAADIQACAAAMhAAIAAAwEMsGwKq6Q1XttZbFAAAAMD0rjQB+KcndkqSq3ldVd16bkgAAAJiGlQLgd5LcvP/96CQ/MPVqAAAAmJqV7gP470leUlXv6bd/r6q+usxzW2vtaZMtDQAAgElaKQA+Icnzkzw8SUvygCRXL/PclkQABAAAWMeWDYCttfOTbEqSqrouySNaa/+2VoUBAAAwWSuNAI46NMly0z8BAADYDYwVAFtrX66qPavqMUnuk+TWSa5IcnaSN7XWrp1ijQAAAEzAWAGwqm6T5N1JfiLJhUkuTnKvJE9K8smqelBr7dJpFQkAAMCuW+k2EKNemGT/JPdsrd2xtXav1todk/x03/7CaRUIAADAZIwbAB+S5GmLF4FprZ2T5OlJfmHShQEAADBZ4wbAmyX55jL7vpnkppMpBwAAgGkZNwB+NMnTquoWo4399tP6/QAAAKxj494G4ilJ3p/koqp6d7pFYG6T5MFJKsnRU6kOAACAiRlrBLC19okkhyc5McmBSR6YLgC+MsnhrbVPTqtAAAAAJmPcEcC01i5LcsIUawEAAGCKxr0GEAAAgN2cAAgAADAQY08BnZWqekS6+wz+QJJXt9bePduKAAAAdk8zGQGsqpOq6pKq+syi9mOq6nNVdUFVnZAkrbW3tNaekOSJSR4zi3oBAGDNVO3aA1awwwBYVTerqmdU1U9O8H1PTnLMovfZI8nLkhyb5Igkx1XVESNP+dN+PwAAADthhwGwtXZ1kmck2W9Sb9paOyvJFYuaj0pyQWvtP1tr1yQ5JcnDq/O8JO9orX18qeNV1eaq2lpVWy+99NJJlQkAADBXxp0C+rEkd59mIUkOSnLRyPa2vu33kvx8kl+qqicu9cLW2omttY2ttY0HHnjglMsEAADYPY27CMxTk/xLVX03yRlJLk7SRp/QWvv2hGtbOO5Lk7x0GscGAAAYknED4Mf6ny9N8pJlnrPHLtbylSS3H9k+uG8DAABgAsYNgL+ZRSN+U3BOksOr6tB0we+xSX5lyu8JAAAwGGMFwNbayZN806p6XZKjkxxQVduSPLO19uqqOj7Ju9KNJp7UWjtvku8LAAAwZKu6EXx/W4Z7pJuqeVJr7WtVdViSi1tr3xz3OK2145ZpPyPdNYarVlWbkmw67LDDdublAAAAc2+sVUCr6pZVdVqSzyT5xyR/meSH+t1/neSZ0ylvfK21La21zRs2bJh1KQAAAOvSuLeBeGGSn0nygCT7JqmRfWdk0U3dAQAAWH/GnQL6qCRPbq29v6oWr/b55SQ/PNmyAAAAmLRxRwD3SXL5Mvv2TfK9yZQDAADAtIwbAM9J8qvL7PulJB+eTDkAAABMy7hTQP8syXuq6l+TvD7dPQEfUlV/mC4A3m9K9Y3NKqAAAAArG2sEsLV2droFYG6W5O/TLQLz7CR3TPLzrbVzplbhmKwCCgAAsLKx7wPYWvtQkvtW1T5JbpXkytbat6dWGQAAABM17jWAo65K8t0k35lwLQAAAEzR2AGwqh5SVR9OFwC/luSqqvpwVf3C1KoDAABgYsYKgFX120m2JPlWkicn+eX+57eSnN7vBwAAYB0b9xrAP0nyqtba7y5qf2VVvTLJM5K8aqKVrZJVQAEAAFY27hTQ/ZO8eZl9b0xy68mUs/OsAgoAALCycQPg+5Pcf5l9909y1mTKAQAAYFqWnQJaVUeMbL40yT9W1f5J3pLkkiS3SfLIJMcm+a0p1ggAAMAErHQN4GeStJHtSvLb/aP12wvemWSPiVcHAADAxKwUAH92zaoAAABg6pYNgK21D6xlIQAAAEzXuLeB+L6q2jPJTRe3t9a+PZGKAAAAmIpxbwS/oapeXlVfTXJVkm8u8ZipqtpUVSdu37591qUAAACsS+OOAJ6c7nYP/5DkgiTXTKugndVa25Jky8aNG58w61oAAADWo3ED4AOS/HZr7XXTLAYAAIDpGfdG8P+VxDV+AAAAu7FxA+BTk/xpVd1hmsUAAAAwPWNNAW2tnVFVP5/kgqq6MMmVSzznqMmWBgAAwCSNFQCr6u+S/EGSc7JOF4EBAAB2UdXOv7a1ydXB1Iy7CMxvJXlGa+250ywGAACA6Rn3GsBvJzl3moUAAAAwXeMGwJck2Vy1K2PC0+VG8AAAACsbdwroAUl+OsnnqurM3HgRmNZae9oE61o1N4IHAABY2bgB8JeSXJtkryQPXGJ/SzLTAAgAAMDKxr0NxKHTLgQAAIDpGvcaQAAAAHZz494H8Hd39JzW2st3vRwAAACmZdxrAP9+hX0Ld3wUAAEAANaxsaaAttZusviR5NZJjkvyySRHTLNIAAAAdt24I4A30lq7MsmpVbUhyauSHD2hmgAAAJiCSSwC86UkGydwHAAAAKZolwJgVd0uyVPShUAAAADWsXFXAb001y/2suCmSfZNclWSR024LgAAACZs3GsAX5YbB8CrkmxL8s7W2uUTrWonVNWmJJsOO+ywWZcCAACwLlVri3Pd7m3jxo1t69atsy7jxqpmXcHa25VzS3+tjv5aHf21Ovpr9YbWZ/pr9fw3uTr6a3Vm1V9zlit2Z1V1bmttyXVaJrEIDAAAALuBZaeAVtX7VnGc1lp7wATqAQAAYEpWugZwnOv6bpfkZ3Lj6wMBAABYZ5YNgK21X15uX1XdIcnTkjw0yWVJXjT50gAAAJikcVcBTZJU1WFJnp7kcUku6X9/VWvtO1OoDQAAgAka9z6AP5bkGUl+OclFSZ6c5KTW2jVTrA0AAIAJWnEV0Kq6R1W9Kcmnktw9yW8lOby19krhDwAAYPey0iqg70jyoCSfTvLY1trr16wqAAAAJm6lKaAP7n8enORlVfWylQ7UWrvNxKoCAABg4lYKgM9esyoAAACYupVuAyEAAgAAzJEVF4EBAABgfsxNAKyqTVV14vbt22ddCgAAwLo0NwGwtbaltbZ5w4YNsy4FAABgXZqbAAgAAMDKBEAAAICBEAABAAAGQgAEAAAYCAEQAABgIARAAACAgdhz1gUAAABzoGrnX9va5OpgRUYAAQAABkIABAAAGAgBEAAAYCAEQAAAgIEQAAEAAAZCAAQAABgIARAAAGAgBEAAAICBEAABAAAGQgAEAAAYCAEQAABgIARAAACAgZibAFhVm6rqxO3bt8+6FAAAgHVpbgJga21La23zhg0bZl0KAADAujQ3ARAAAICVCYAAAAADIQACAAAMhAAIAAAwEAIgAADAQAiAAAAAAyEAAgAADIQACAAAMBACIAAAwEAIgAAAAAMhAAIAAAyEAAgAADAQAiAAAMBACIAAAAADIQACAAAMhAAIAAAwEAIgAADAQAiAAAAAAyEAAgAADIQACAAAMBACIAAAwEAIgAAAAAMhAAIAAAyEAAgAADAQAiAAAMBACIAAAAADIQACAAAMhAAIAAAwEAIgAADAQAiAAAAAA7GuA2BV3bGqXl1Vb5h1LQAAALu7NQ+AVXVSVV1SVZ9Z1H5MVX2uqi6oqhOSpLX2n621x691jQAAAPNoFiOAJyc5ZrShqvZI8rIkxyY5IslxVXXE2pcGAAAwv9Y8ALbWzkpyxaLmo5Jc0I/4XZPklCQPH/eYVbW5qrZW1dZLL710gtUCAADMj/VyDeBBSS4a2d6W5KCq2r+qXpnkblX19OVe3Fo7sbW2sbW28cADD5x2rQAAALulPWddwEpaa5cneeKs6wAAAJgH62UE8CtJbj+yfXDfBgAAwISslwB4TpLDq+rQqrppkscmOX3GNQEAAMyVWdwG4nVJPpLkTlW1raoe31q7NsnxSd6V5LNJTmutnbfWtQEAAMyzNb8GsLV23DLtZyQ5Y2ePW1Wbkmw67LDDdvYQAAAAc229TAHdZa21La21zRs2bJh1KQAAAOvS3ARAAAAAViYAAgAADIQACAAAMBBzEwCralNVnbh9+/ZZlwIAALAuzU0AtAgMAADAyuYmAAIAALAyARAAAGAgBEAAAICBEAABAAAGQgAEAAAYiLkJgG4DAQAAu6mqXXswtrkJgG4DAQAAsLK5CYAAAACsTAAEAAAYCAEQAABgIARAAACAgRAAAQAABkIABAAAGIi5CYDuAwgAALCyuQmA7gMIAACwsrkJgAAAAKxMAAQAABgIARAAAGAgBEAAAICBEAABAAAGQgAEAAAYCAEQAABgIOYmALoRPAAADFTVzj8GZm4CoBvBAwAArGxuAiAAAAArEwABAAAGQgAEAAAYCAEQAABgIARAAACAgRAAAQAABkIABAAAGAgBEAAAYCAEQAAAgIEQAAEAAAZibgJgVW2qqhO3b98+61IAAADWpbkJgK21La21zRs2bJh1KQAAAOvS3ARAAAAAViYAAgAADIQACAAAMBACIAAAwEAIgAAAAAMhAAIAAAyEAAgAADAQAiAAAMBACIAAAAADIQACAAAMhAAIAAAwEAIgAADAQMxNAKyqTVV14vbt22ddCgAAwLo0NwGwtbaltbZ5w4YNsy4FAABgXZqbAAgAAMDKBEAAAICBEAABAAAGQgAEAAAYCAEQAABgIARAAACAgRAAAQAABkIABAAAGAgBEAAAYCAEQAAAgIEQAAEAAAZCAAQAABgIARAAAGAgBEAAAICBEAABAAAGQgAEAAAYCAEQAABgIARAAACAgRAAAQAABmJuAmBVbaqqE7dv3z7rUgAAANaluQmArbUtrbXNGzZsmHUpAAAA69LcBEAAAABWJgACAAAMhAAIAAAwEAIgAADAQAiAAAAAAyEAAgAADIQACAAAMBACIAAAwEAIgAAAAAMhAAIAAAyEAAgAADAQAiAAAMBACIAAAAADIQACAAAMhAAIAAAwEAIgAADAQAiAAAAAAyEAAgAADIQACAAAMBACIAAAwEAIgAAAAAMhAAIAAAyEAAgAADAQAiAAAMBACIAAAAADIQACAAAMhAAIAAAwEAIgAADAQAiAAAAAAyEAAgAADMSesy5gJVV1iyQvT3JNkjNba6+dcUkAAAC7rTUfAayqk6rqkqr6zKL2Y6rqc1V1QVWd0Dc/KskbWmtPSPKwta4VAABgnsxiCujJSY4ZbaiqPZK8LMmxSY5IclxVHZHk4CQX9U/73hrWCAAAMHfWPAC21s5KcsWi5qOSXNBa+8/W2jVJTkny8CTb0oXAZIVaq2pzVW2tqq2XXnrpNMoGAADmUdXOP3ZD62URmINy/Uhf0gW/g5K8KckvVtUrkmxZ7sWttRNbaxtbaxsPPPDA6VYKAACwm1rXi8C01v4nyW/Mug4AAIB5sF5GAL+S5PYj2wf3bQAAAEzIegmA5yQ5vKoOraqbJnlsktNnXBMAAMBcmcVtIF6X5CNJ7lRV26rq8a21a5Mcn+RdST6b5LTW2nmrPO6mqjpx+/btky8aAABgDlRrbdY1TNTGjRvb1q1bZ13Gje2mqwTtkl05t/TX6uiv1dFfq6O/Vm9ofaa/Vs9/k6ujv1ZHf62ddZqlqurc1trGpfatlymgAAAATJkACAAAMBACIAAAwEAIgAAAAAMxNwHQKqAAAAArm5sA2Frb0lrbvGHDhlmXAgAAsC7NTQAEAABgZQIgAADAQAiAAAAAAyEAAgAADIQACAAAMBBzEwDdBgIAAGBlcxMA3QYCAABgZXMTAAEAAFiZAAgAADAQAiAAAMBACIAAAAADIQACAAAMRLXWZl3DRFXVpUm+PKHDHZDksgkdi9XR97Oj72dH38+W/p8dfT87+n529P3sDKHvf7i1duBSO+YuAE5SVW1trW2cdR1DpO9nR9/Pjr6fLf0/O/p+dvT97Oj72Rl635sCCgAAMBACIAAAwEAIgCs7cdYFDJi+nx19Pzv6frb0/+zo+9nR97Oj72dn0H3vGkAAAICBMAIIAAAwEAIgAADAQAiAy6iqY6rqc1V1QVWdMOt6hqSqLqyqT1fVJ6pq66zrmWdVdVJVXVJVnxlpu3VVvaeqvtD/vNUsa5xXy/T9s6rqK/25/4mqesgsa5xXVXX7qnp/Vf1HVZ1XVU/u2537U7ZC3zv3p6yq9q6qf6uqT/Z9/+y+/dCq+lj/986pVXXTWdc6b1bo+5Or6ksj5/2RMy51blXVHlX171X1tn570Oe9ALiEqtojycuSHJvkiCTHVdURs61qcH62tXbkkO/RskZOTnLMorYTkry3tXZ4kvf220zeyblx3yfJi/pz/8jW2hlrXNNQXJvkKa21I5LcM8mT+v+Nd+5P33J9nzj3p+3qJD/XWvvJJEcmOaaq7pnkeen6/rAkX0/y+NmVOLeW6/sk+eOR8/4TsypwAJ6c5LMj24M+7wXApR2V5ILW2n+21q5JckqSh8+4Jpi41tpZSa5Y1PzwJP/U//5PSR6xljUNxTJ9zxporX21tfbx/vdvpvuj4KA496duhb5nylrnW/3mXv2jJfm5JG/o2533U7BC37MGqurgJL+Q5B/77crAz3sBcGkHJbloZHtb/B/UWmpJ3l1V51bV5lkXM0C3ba19tf/9a0luO8tiBuj4qvpUP0XUFMQpq6pDktwtycfi3F9Ti/o+ce5PXT8N7hNJLknyniRfTHJla+3a/in+3pmSxX3fWls475/Tn/cvqqqbza7CufbiJE9Ncl2/vX8Gft4LgKxH92mt3T3dFNwnVdX9Zl3QULXuPjH+lXLtvCLJj6SbIvTVJC+YaTVzrqpumeSNSf6gtfaN0X3O/elaou+d+2ugtfa91tqRSQ5ON9vpzrOtaDgW931V3TXJ09N9Bz+V5NZJnja7CudTVT00ySWttXNnXct6IgAu7StJbj+yfXDfxhporX2l/3lJkjen+z8p1s7FVXW7JOl/XjLjegajtXZx/0fCdUn+Ic79qamqvdIFkNe21t7UNzv318BSfe/cX1uttSuTvD/JvZLsV1V79rv8vTNlI31/TD8lurXWrk7ymjjvp+HeSR5WVRemu6Tr55K8JAM/7wXApZ2T5PB+haCbJnlsktNnXNMgVNUtqmrfhd+TPCjJZ1Z+FRN2epJf63//tSRvnWEtg7IQPnqPjHN/KvrrP16d5LOttReO7HLuT9lyfe/cn76qOrCq9ut/3yfJA9Ndg/n+JL/UP815PwXL9P35I//gVOmuQXPeT1hr7emttYNba4ek+3v+fa21/5WBn/fVzXJhsX4J6hcn2SPJSa2158y2omGoqjumG/VLkj2T/Iu+n56qel2So5MckOTiJM9M8pYkpyW5Q5IvJ3l0a81iJRO2TN8fnW4KXEtyYZLfHrkmjQmpqvskOTvJp3P9NSF/ku5aNOf+FK3Q98fFuT9VVfUT6Ra72CPdAMBprbW/6P9/95R0UxD/Pcnj+hEpJmSFvn9fkgOTVJJPJHniyGIxTFhVHZ3kj1prDx36eS8AAgAADIQpoAAAAAMhAAIAAAyEAAgAADAQAiAAAMBACIAAAAADIQACMHFV9ayqalX1riX2vaGqzlzDWo7ua7nrWr3nalTVXarq7Kr6n77OQ2Zd01Kq6pZ9fb8+61oA2HkCIADT9KCq+qlZF7HOPT/JfkkeluReSdz/DoCpEQABmJYr0t3w+xmzLmSaqmrvXTzEnZO8p7X23tbaR2d5M+Lq7OrnAWAdEwABmJaW5DlJHlZVP77ck/rpopct0d6q6viR7Qur6u+q6oSq+mpVba+qF/Sh5SFVdV5VfbOq3lJVt1rirX6oqt7WT7X8r6p64hLved+q+kBVfbuqLq+qf6iqfUf2/3pf11FVdWZVfSfJH6/w2Y6sqvf2x/t6Vb22qm7b7zukqlqSH0nyh/1xz1zmOP9UVe8e2b5T//w3jbTdo287fKTt+Kr6QlVdXVUXVNUfLjrus6rqsqq6T1Wdk+SqJL/c7/vFqvp8VX2nqs5KF1QX1/Wwqjq379OvV9XHqur+y/UHALMnAAIwTa9P8oVMbhTwsUmOSvIbSf42yf9J8sIkf5nkz5I8Mcn9kzx3ide+OsmnkjwqyRlJXlFVD13YWVX3TvKvSb6W5JeS/EGShyR5zRLHel2SLf3+ty1VaFUdmOTMJDdP8itJfq+v7T1VddN0Uz3v1b/fv/S//+4yn/vsJPeqqj367fulC2v3GXnO/ZJc3Fr7Qv/+T0jyf5OcnmRTuu/iBVV1wqJj3zzJPyX5xyTHJPm3qrp7klOTfDJdf21Jctqiz/cjSd6Q5H398f9X3xe3XuYzALAO7DnrAgCYX62166rquUleXVV/3lr7/C4e8qokv9xa+16Sd1bVw9MFq8Nba19Kkqr6ySS/li4MjnpHa+1P+t/f1QeYP831Ae5vkny4tfaYhRdU1VeSvLeq7tpa+8zIsV7aWnvJDmp9Sv/zwa21b/TH+0KSjyb5xdba65J8tKquTvLV1tpHVzjW2UlumeRuSbYmuW+60Pb4qrpza+38vu3s/n1ukuRZSU5urS3U8e6q2pDk6VX14tbaVX37Pkn+T2vtrSOf+7Qkn0/y6NZaS/KOPrT+1UhNd0vyzdba6AjoGTvoEwBmzAggANP2/yX5ryRPn8CxzuzD34ILkly4EP5G2g7sA8uoNy/aflOSe1TVHlV183QjcKdV1Z4LjyQfTPLdJPdY9Nq3j1HrUUnevRD+kqS19rEkF+aGI3c71Fr7XJJL0oW8pBvte0eSj4+03Sd9AExycJIfSjfqN+rUJD+QZHRKbuuPtbj20/vwt+BNi57z6SQb+umpD6qqW6zmMwEwGwIgAFPVWrs23XTNx1XVD+/i4a5ctH3NMm2VZHEAvGSJ7T2THJDkVkn2SPLydIFv4XF1kr2S3H7Ray8eo9bbLfO8i7Nz0yTPTnLfqrp9kjukC6cLbXdJcmCuD4C3W6bOhe3R9/96a+2aRc/7wSzdX9/Xh9KHJ7ljupG/y6rqX/qprwCsUwIgAGvhpHQB4mlL7Lsqi8LaMou47KrbLLF9bZLL0oXIluSZSX5qicdJi17bsmNfXeI9k+S26VZIXa2z043y3S/Jf7TWLu/b7tu3fSPdNXsL750l3v+2/c/R91/qs3xtidfe6LO01t7eWrtvkv2TPD7Jz6e77hCAdUoABGDq+lsb/F2S38z1o1MLtiXZt6oOGml70BTKeOQS2+e21r7XWvufdNfm3am1tnWJx3/vxPt9LMmDF60i+lNJDkk3erdaZ6Ub5dvc/550AfCH0y0y8+HW2nV9+7Yk/51+Rc8Rj04XFD+9g/c6J93qrTXS9qjlntxa295a+5d002yP2PFHAWBWLAIDwFp5VZI/SfIzST4w0v7OJN9JclJVvSDJobnxAi6TcGxVPad/70cleWC6KYwLnppuwZfr0q1u+c10Uy1/IckzdmIBmxcm+Z10C848L90iLn+TLny9cSfq/2S68Ha/JK9IktbaFVX1H33b91da7RffeVaSV1XV5Unek24F0t9J8icjC8As53npAuxpVfXqJHdNN8L3fVX12+mum3xnurB5eLrA+f924rMBsEaMAAKwJlpr307yoiXaL0vyi+kWLnlLkselG9GatN9Kcvf+PR6a5EmttdNH6vhguiB1YJJ/Tnfrg6cmuSjjXfN3A621S5P8bLoprq9L8rJ0I3YPXOKau3GOd12SD/ebZ43sWrju74OLnv8PSZ6cbqTzbUmOS/KU1trfjPFeW9PdcuNu6frrEUkes+hpn0rXVy9M8u50K6r+Q5ae5gvAOlE3XOALAACAeWUEEAAAYCAEQAAAgIEQAAEAAAZCAAQAABgIARAAAGAgBEAAAICBEAABAAAGQgAEAAAYiP8fC03RrzAbU6MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(15, 10))\n",
    "plt.hist(train_word_counts, bins=50, facecolor='r',label='train')\n",
    "plt.title('Log-Histogram of word count in review', fontsize=15)\n",
    "plt.yscale('log', nonposy='clip')\n",
    "plt.legend()\n",
    "plt.xlabel('Number of words', fontsize=15)\n",
    "plt.ylabel('Number of reviews', fontsize=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "리뷰 단어 개수 최대 값: 41\n",
      "리뷰 단어 개수 최소 값: 1\n",
      "리뷰 단어 개수 평균 값: 7.58\n",
      "리뷰 단어 개수 표준편차: 6.51\n",
      "리뷰 단어 개수 중간 값: 6.0\n",
      "리뷰 단어 개수 제 1 사분위: 3.0\n",
      "리뷰 단어 개수 제 3 사분위: 9.0\n"
     ]
    }
   ],
   "source": [
    "print('리뷰 단어 개수 최대 값: {}'.format(np.max(train_word_counts)))\n",
    "print('리뷰 단어 개수 최소 값: {}'.format(np.min(train_word_counts)))\n",
    "print('리뷰 단어 개수 평균 값: {:.2f}'.format(np.mean(train_word_counts)))\n",
    "print('리뷰 단어 개수 표준편차: {:.2f}'.format(np.std(train_word_counts)))\n",
    "print('리뷰 단어 개수 중간 값: {}'.format(np.median(train_word_counts)))\n",
    "# 사분위의 대한 경우는 0~100 스케일로 되어있음\n",
    "print('리뷰 단어 개수 제 1 사분위: {}'.format(np.percentile(train_word_counts, 25)))\n",
    "print('리뷰 단어 개수 제 3 사분위: {}'.format(np.percentile(train_word_counts, 75)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "물음표가있는 질문: 8.25%\n",
      "마침표가 있는 질문: 51.76%\n"
     ]
    }
   ],
   "source": [
    "qmarks = np.mean(train_data['document'].astype(str).apply(lambda x: '?' in x)) # 물음표가 구두점으로 쓰임\n",
    "fullstop = np.mean(train_data['document'].astype(str).apply(lambda x: '.' in x)) # 마침표\n",
    "                  \n",
    "print('물음표가있는 질문: {:.2f}%'.format(qmarks * 100))\n",
    "print('마침표가 있는 질문: {:.2f}%'.format(fullstop * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 데이터 전처리"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import json\n",
    "from konlpy.tag import Okt\n",
    "from tensorflow.python.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.python.keras.preprocessing.text import Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         id                                           document  label\n",
      "0   9976970                                아 더빙.. 진짜 짜증나네요 목소리      0\n",
      "1   3819312                  흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나      1\n",
      "2  10265843                                  너무재밓었다그래서보는것을추천한다      0\n",
      "3   9045019                      교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정      0\n",
      "4   6483659  사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...      1\n"
     ]
    }
   ],
   "source": [
    "DATA_IN_PATH ='./data_in/'\n",
    "\n",
    "train_data = pd.read_csv(DATA_IN_PATH + 'ratings_train.txt', header=0, delimiter='\\t', quoting=3 )\n",
    "\n",
    "print(train_data.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "아 더빙 진짜 짜증나네요 목소리\n"
     ]
    }
   ],
   "source": [
    "review_text = re.sub(\"[^가-힣ㄱ-ㅎㅏ-ㅣ\\\\s]\", \"\", train_data['document'][0]) \n",
    "print(review_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['아', '더빙', '진짜', '짜증나다', '목소리']\n"
     ]
    }
   ],
   "source": [
    "okt=Okt()\n",
    "review_text = okt.morphs(review_text, stem=True)\n",
    "print(review_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['더빙', '진짜', '짜증나다', '목소리']\n"
     ]
    }
   ],
   "source": [
    "stop_words = set(['은', '는', '이', '가', '하', '아', '것', '들','의', '있', '되', '수', '보', '주', '등', '한'])\n",
    "clean_review = [token for token in review_text if not token in stop_words]\n",
    "print(clean_review)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "stop_words = set(['은', '는', '이', '가', '하', '아', '것', '들','의', '있', '되', '수', '보', '주', '등', '한'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "def preprocessing(review, okt, remove_stopwords = False, stop_words = []):\n",
    "    # 함수의 인자는 다음과 같다.\n",
    "    # review : 전처리할 텍스트\n",
    "    # okt : okt 객체를 반복적으로 생성하지 않고 미리 생성후 인자로 받는다.\n",
    "    # remove_stopword : 불용어를 제거할지 선택 기본값은 False\n",
    "    # stop_word : 불용어 사전은 사용자가 직접 입력해야함 기본값은 비어있는 리스트\n",
    "    \n",
    "    # 1. 한글 및 공백을 제외한 문자 모두 제거.\n",
    "    review_text = re.sub(\"[^가-힣ㄱ-ㅎㅏ-ㅣ\\\\s]\", \"\", review)\n",
    "    \n",
    "    # 2. okt 객체를 활용해서 형태소 단위로 나눈다.\n",
    "    word_review = okt.morphs(review_text, stem=True)\n",
    "    \n",
    "    if remove_stopwords:\n",
    "        \n",
    "        # 불용어 제거(선택적)\n",
    "        word_review = [token for token in word_review if not token in stop_words]\n",
    "        \n",
    "   \n",
    "    return word_review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "stop_words = [ '은', '는', '이', '가', '하', '아', '것', '들','의', '있', '되', '수', '보', '주', '등', '한']\n",
    "okt = Okt()\n",
    "clean_train_review = []\n",
    "\n",
    "for review in train_data['document']:\n",
    "    # 비어있는 데이터에서 멈추지 않도록 string인 경우만 진행\n",
    "    if type(review) == str:\n",
    "        clean_train_review.append(preprocessing(review, okt, remove_stopwords = True, stop_words=stop_words))\n",
    "    else:\n",
    "        clean_train_review.append([])  #string이 아니면 비어있는 값 추가"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "test_data = pd.read_csv(DATA_IN_PATH + 'ratings_test.txt', header=0, delimiter='\\t', quoting=3 )\n",
    "\n",
    "clean_test_review = []\n",
    "\n",
    "for review in test_data['document']:\n",
    "    # 비어있는 데이터에서 멈추지 않도록 string인 경우만 진행\n",
    "    if type(review) == str:\n",
    "        clean_test_review.append(preprocessing(review, okt, remove_stopwords = True, stop_words=stop_words))\n",
    "    else:\n",
    "        clean_test_review.append([])  #string이 아니면 비어있는 값 추가"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(clean_train_review)\n",
    "train_sequences = tokenizer.texts_to_sequences(clean_train_review)\n",
    "test_sequences = tokenizer.texts_to_sequences(clean_test_review)\n",
    "\n",
    "word_vocab = tokenizer.word_index # 단어 사전 형태\n",
    "word_vocab[\"<PAD>\"] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 8 # 문장 최대 길이\n",
    "\n",
    "train_inputs = pad_sequences(train_sequences, maxlen=MAX_SEQUENCE_LENGTH, padding='post') # 학습 데이터를 벡터화\n",
    "train_labels = np.array(train_data['label']) # 학습 데이터의 라벨\n",
    "\n",
    "test_inputs = pad_sequences(test_sequences, maxlen=MAX_SEQUENCE_LENGTH, padding='post') # 테스트 데이터를 벡터화\n",
    "test_labels = np.array(test_data['label']) # 테스트 데이터의 라벨"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "DATA_IN_PATH = './data_in/'\n",
    "TRAIN_INPUT_DATA = 'nsmc_train_input.npy'\n",
    "TRAIN_LABEL_DATA = 'nsmc_train_label.npy'\n",
    "TEST_INPUT_DATA = 'nsmc_test_input.npy'\n",
    "TEST_LABEL_DATA = 'nsmc_test_label.npy'\n",
    "DATA_CONFIGS = 'data_configs.json'\n",
    "\n",
    "data_configs = {}\n",
    "\n",
    "data_configs['vocab'] = word_vocab\n",
    "data_configs['vocab_size'] = len(word_vocab) # vocab size 추가\n",
    "\n",
    "import os\n",
    "# 저장하는 디렉토리가 존재하지 않으면 생성\n",
    "if not os.path.exists(DATA_IN_PATH):\n",
    "    os.makedirs(DATA_IN_PATH)\n",
    "\n",
    "# 전처리 된 학습 데이터를 넘파이 형태로 저장\n",
    "np.save(open(DATA_IN_PATH + TRAIN_INPUT_DATA, 'wb'), train_inputs)\n",
    "np.save(open(DATA_IN_PATH + TRAIN_LABEL_DATA, 'wb'), train_labels)\n",
    "# 전처리 된 테스트 데이터를 넘파이 형태로 저장\n",
    "np.save(open(DATA_IN_PATH + TEST_INPUT_DATA, 'wb'), test_inputs)\n",
    "np.save(open(DATA_IN_PATH + TEST_LABEL_DATA, 'wb'), test_labels)\n",
    "\n",
    "# 데이터 사전을 json 형태로 저장\n",
    "json.dump(data_configs, open(DATA_IN_PATH + DATA_CONFIGS, 'w'), ensure_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}