6.CHATBOT/6.3.preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from preprocess import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = 'data_in/ChatBotData.csv_short'\n",
    "VOCAB_PATH = 'data_in/vocabulary.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs, outputs = load_data(PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "char2idx, idx2char, vocab_size = load_vocabulary(PATH, VOCAB_PATH, tokenize_as_morph=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "index_inputs, input_seq_len = enc_processing(inputs, char2idx, tokenize_as_morph=False)\n",
    "index_outputs, output_seq_len = dec_output_processing(outputs, char2idx, tokenize_as_morph=False)\n",
    "index_targets = dec_target_processing(outputs, char2idx, tokenize_as_morph=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_configs = {}\n",
    "data_configs['char2idx'] = char2idx\n",
    "data_configs['idx2char'] = idx2char\n",
    "data_configs['vocab_size'] = vocab_size\n",
    "data_configs['pad_symbol'] = PAD\n",
    "data_configs['std_symbol'] = STD\n",
    "data_configs['end_symbol'] = END\n",
    "data_configs['unk_symbol'] = UNK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_IN_PATH = './data_in/'\n",
    "TRAIN_INPUTS = 'train_inputs.npy'\n",
    "TRAIN_OUTPUTS = 'train_outputs.npy'\n",
    "TRAIN_TARGETS = 'train_targets.npy'\n",
    "DATA_CONFIGS = 'data_configs.json'\n",
    "\n",
    "np.save(open(DATA_IN_PATH + TRAIN_INPUTS, 'wb'), index_inputs)\n",
    "np.save(open(DATA_IN_PATH + TRAIN_OUTPUTS , 'wb'), index_outputs)\n",
    "np.save(open(DATA_IN_PATH + TRAIN_TARGETS , 'wb'), index_targets)\n",
    "\n",
    "json.dump(data_configs, open(DATA_IN_PATH + DATA_CONFIGS, 'w'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'<PAD>': 0,\n",
       " '<SOS>': 1,\n",
       " '<END>': 2,\n",
       " '<UNK>': 3,\n",
       " '다시': 4,\n",
       " '거예요': 5,\n",
       " '싶어': 6,\n",
       " '따라': 7,\n",
       " '가상화폐': 8,\n",
       " '남자친구': 9,\n",
       " '끄고': 10,\n",
       " '켜고': 11,\n",
       " '집에': 12,\n",
       " '나오세요': 13,\n",
       " '믿어줘': 14,\n",
       " '가만': 15,\n",
       " '사람도': 16,\n",
       " '운동만': 17,\n",
       " '집착하지': 18,\n",
       " '나왔다': 19,\n",
       " '따뜻하게': 20,\n",
       " '것': 21,\n",
       " '잊고': 22,\n",
       " '혼자를': 23,\n",
       " '운동을': 24,\n",
       " '빠를수록': 25,\n",
       " '같아요': 26,\n",
       " '열': 27,\n",
       " '그': 28,\n",
       " '해보세요': 29,\n",
       " '돈은': 30,\n",
       " '같아': 31,\n",
       " '절약해봐요': 32,\n",
       " '필요한': 33,\n",
       " '오늘': 34,\n",
       " '그럴': 35,\n",
       " '해': 36,\n",
       " '나갔어': 37,\n",
       " '나라를': 38,\n",
       " '가스불': 39,\n",
       " '함께': 40,\n",
       " '생일인데': 41,\n",
       " '뭐가': 42,\n",
       " '구하셨나요': 43,\n",
       " '가난한': 44,\n",
       " '더': 45,\n",
       " '질린다': 46,\n",
       " '감기': 47,\n",
       " '사세요': 48,\n",
       " '땀난다': 49,\n",
       " '운동': 50,\n",
       " '가스비': 51,\n",
       " '자의': 52,\n",
       " '빨리': 53,\n",
       " '좋다': 54,\n",
       " '새출발': 55,\n",
       " '생각해보세요': 56,\n",
       " '식혀주세요': 57,\n",
       " '설득해보세요': 58,\n",
       " '선물로': 59,\n",
       " '뭘': 60,\n",
       " '갔어': 61,\n",
       " '다음': 62,\n",
       " '마세요': 63,\n",
       " '땀을': 64,\n",
       " '나온거': 65,\n",
       " '돌아가서': 66,\n",
       " '안': 67,\n",
       " '켜놓고': 68,\n",
       " '있어도': 69,\n",
       " '때까지': 70,\n",
       " '마음을': 71,\n",
       " '보인다': 72,\n",
       " '승진': 73,\n",
       " '줄까': 74,\n",
       " '망함': 75,\n",
       " '어서': 76,\n",
       " '궁금해': 77,\n",
       " '잘생겼어': 78,\n",
       " '필요했던': 79,\n",
       " '설움': 80,\n",
       " '달에는': 81,\n",
       " '좀': 82,\n",
       " '평소에': 83,\n",
       " '즐기세요': 84,\n",
       " '거짓말': 85,\n",
       " '너무': 86,\n",
       " '데려가고': 87,\n",
       " '적당히': 88,\n",
       " '또': 89,\n",
       " '나': 90,\n",
       " '전생에': 91,\n",
       " '쫄딱': 92,\n",
       " '혼자인게': 93,\n",
       " '하세요': 94,\n",
       " '많이': 95,\n",
       " '가끔은': 96,\n",
       " '좋을': 97,\n",
       " '게': 98,\n",
       " '남자친구가': 99,\n",
       " '결단은': 100,\n",
       " '교회': 101,\n",
       " '가끔': 102,\n",
       " '걸리겠어': 103,\n",
       " '좋을까': 104,\n",
       " '들어올': 105,\n",
       " '바빠': 106,\n",
       " '비싼데': 107,\n",
       " '훈훈해': 108,\n",
       " '좋아요': 109,\n",
       " '뭐하는지': 110}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char2idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: '<PAD>',\n",
       " 1: '<SOS>',\n",
       " 2: '<END>',\n",
       " 3: '<UNK>',\n",
       " 4: '다시',\n",
       " 5: '거예요',\n",
       " 6: '싶어',\n",
       " 7: '따라',\n",
       " 8: '가상화폐',\n",
       " 9: '남자친구',\n",
       " 10: '끄고',\n",
       " 11: '켜고',\n",
       " 12: '집에',\n",
       " 13: '나오세요',\n",
       " 14: '믿어줘',\n",
       " 15: '가만',\n",
       " 16: '사람도',\n",
       " 17: '운동만',\n",
       " 18: '집착하지',\n",
       " 19: '나왔다',\n",
       " 20: '따뜻하게',\n",
       " 21: '것',\n",
       " 22: '잊고',\n",
       " 23: '혼자를',\n",
       " 24: '운동을',\n",
       " 25: '빠를수록',\n",
       " 26: '같아요',\n",
       " 27: '열',\n",
       " 28: '그',\n",
       " 29: '해보세요',\n",
       " 30: '돈은',\n",
       " 31: '같아',\n",
       " 32: '절약해봐요',\n",
       " 33: '필요한',\n",
       " 34: '오늘',\n",
       " 35: '그럴',\n",
       " 36: '해',\n",
       " 37: '나갔어',\n",
       " 38: '나라를',\n",
       " 39: '가스불',\n",
       " 40: '함께',\n",
       " 41: '생일인데',\n",
       " 42: '뭐가',\n",
       " 43: '구하셨나요',\n",
       " 44: '가난한',\n",
       " 45: '더',\n",
       " 46: '질린다',\n",
       " 47: '감기',\n",
       " 48: '사세요',\n",
       " 49: '땀난다',\n",
       " 50: '운동',\n",
       " 51: '가스비',\n",
       " 52: '자의',\n",
       " 53: '빨리',\n",
       " 54: '좋다',\n",
       " 55: '새출발',\n",
       " 56: '생각해보세요',\n",
       " 57: '식혀주세요',\n",
       " 58: '설득해보세요',\n",
       " 59: '선물로',\n",
       " 60: '뭘',\n",
       " 61: '갔어',\n",
       " 62: '다음',\n",
       " 63: '마세요',\n",
       " 64: '땀을',\n",
       " 65: '나온거',\n",
       " 66: '돌아가서',\n",
       " 67: '안',\n",
       " 68: '켜놓고',\n",
       " 69: '있어도',\n",
       " 70: '때까지',\n",
       " 71: '마음을',\n",
       " 72: '보인다',\n",
       " 73: '승진',\n",
       " 74: '줄까',\n",
       " 75: '망함',\n",
       " 76: '어서',\n",
       " 77: '궁금해',\n",
       " 78: '잘생겼어',\n",
       " 79: '필요했던',\n",
       " 80: '설움',\n",
       " 81: '달에는',\n",
       " 82: '좀',\n",
       " 83: '평소에',\n",
       " 84: '즐기세요',\n",
       " 85: '거짓말',\n",
       " 86: '너무',\n",
       " 87: '데려가고',\n",
       " 88: '적당히',\n",
       " 89: '또',\n",
       " 90: '나',\n",
       " 91: '전생에',\n",
       " 92: '쫄딱',\n",
       " 93: '혼자인게',\n",
       " 94: '하세요',\n",
       " 95: '많이',\n",
       " 96: '가끔은',\n",
       " 97: '좋을',\n",
       " 98: '게',\n",
       " 99: '남자친구가',\n",
       " 100: '결단은',\n",
       " 101: '교회',\n",
       " 102: '가끔',\n",
       " 103: '걸리겠어',\n",
       " 104: '좋을까',\n",
       " 105: '들어올',\n",
       " 106: '바빠',\n",
       " 107: '비싼데',\n",
       " 108: '훈훈해',\n",
       " 109: '좋아요',\n",
       " 110: '뭐하는지'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx2char"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}