file2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import string\n",
    "tqdm.pandas()\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train shape :  (51, 110)\n",
      "Test shape :  (26, 110)\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv(\"cleaned_train.csv\")\n",
    "test = pd.read_csv(\"cleaned_test.csv\")\n",
    "print(\"Train shape : \",train.shape)\n",
    "print(\"Test shape : \",test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_vocab(sentences, verbose =  True):\n",
    "    \"\"\"\n",
    "    :param sentences: list of list of words\n",
    "    :return: dictionary of words and their count\n",
    "    \"\"\"\n",
    "    vocab = {}\n",
    "    for sentence in tqdm(sentences, disable = (not verbose)):\n",
    "        for word in sentence.split():\n",
    "            try:\n",
    "                vocab[word] += 1\n",
    "            except KeyError:\n",
    "                vocab[word] = 1\n",
    "    return vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 51/51 [00:00<00:00, 4260.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'20': 11, 'bonus': 4, 'for': 286, 'registered': 3, 'developers': 10}\n"
     ]
    }
   ],
   "source": [
    "vocab = build_vocab(train['text'])\n",
    "print({k: vocab[k] for k in list(vocab)[:5]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_FILE = 'glove.840B.300d.txt'\n",
    "def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n",
    "embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding = 'utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import operator \n",
    "\n",
    "def check_coverage(vocab,embeddings_index):\n",
    "    a = {}\n",
    "    oov = {}\n",
    "    k = 0\n",
    "    i = 0\n",
    "    for word in tqdm(vocab):\n",
    "        try:\n",
    "            a[word] = embeddings_index[word]\n",
    "            k += vocab[word]\n",
    "        except:\n",
    "\n",
    "            oov[word] = vocab[word]\n",
    "            i += vocab[word]\n",
    "            pass\n",
    "\n",
    "    print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n",
    "    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n",
    "    sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n",
    "\n",
    "    return sorted_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5731/5731 [00:00<00:00, 718332.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 0.00% of vocab\n",
      "Found embeddings for  0.00% of all text\n"
     ]
    }
   ],
   "source": [
    "oov = check_coverage(vocab,embeddings_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('.', 1848),\n",
       " ('the', 1414),\n",
       " ('in', 487),\n",
       " ('is', 398),\n",
       " ('you', 370),\n",
       " ('that', 358),\n",
       " ('i', 337),\n",
       " ('it', 314),\n",
       " ('for', 286),\n",
       " ('with', 271),\n",
       " ('we', 218),\n",
       " ('on', 213),\n",
       " ('as', 212),\n",
       " ('this', 196),\n",
       " ('be', 196),\n",
       " ('are', 188),\n",
       " ('can', 164),\n",
       " ('your', 158),\n",
       " ('have', 155),\n",
       " ('my', 148),\n",
       " (')', 145),\n",
       " ('(', 145),\n",
       " ('but', 140),\n",
       " ('at', 136),\n",
       " ('or', 135),\n",
       " ('from', 126),\n",
       " ('will', 120),\n",
       " ('all', 120),\n",
       " ('our', 120),\n",
       " ('was', 116),\n",
       " ('not', 116),\n",
       " ('its', 107),\n",
       " ('they', 102),\n",
       " ('an', 100),\n",
       " ('so', 98),\n",
       " ('he', 97),\n",
       " ('if', 91),\n",
       " ('what', 90),\n",
       " ('his', 87),\n",
       " ('by', 87),\n",
       " ('more', 86),\n",
       " ('time', 83),\n",
       " ('their', 82),\n",
       " ('would', 80),\n",
       " ('—', 79),\n",
       " ('about', 79),\n",
       " ('how', 78),\n",
       " ('when', 75),\n",
       " ('out', 74),\n",
       " ('do', 72)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oov[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#for punct in string.punctuation:\n",
    "#    print(punct,punct in embeddings_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "to_remove = ['a','to','of','and']\n",
    "def remove_words(x):\n",
    "    x = str(x)\n",
    "    y = []\n",
    "    for w in x.split():\n",
    "        if w not in to_remove:\n",
    "            y.append(w)    \n",
    "    return \" \".join(y)\n",
    "\n",
    "def remove_punc(x):\n",
    "    x = str(x)\n",
    "    for punct in '\\n' + '\\r':\n",
    "        x = x.replace(punct, ' ')\n",
    "    for punct in string.punctuation:\n",
    "        x = x.replace(punct, f' {punct} ')   \n",
    "    return x.lower()\n",
    "\n",
    "def clean_text(x):\n",
    "    x = remove_punc(x)\n",
    "    x = remove_words(x)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 51/51 [00:00<00:00, 3007.90it/s]\n",
      "100%|██████████| 26/26 [00:00<00:00, 2172.57it/s]\n",
      "100%|██████████| 51/51 [00:00<00:00, 5115.00it/s]\n"
     ]
    }
   ],
   "source": [
    "train[\"text\"] = train[\"text\"].progress_apply(lambda x: clean_text(x))\n",
    "test[\"text\"] = test[\"text\"].progress_apply(lambda x: clean_text(x))\n",
    "vocab = build_vocab(train[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5731/5731 [00:00<00:00, 718396.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found embeddings for 0.00% of vocab\n",
      "Found embeddings for  0.00% of all text\n"
     ]
    }
   ],
   "source": [
    "oov = check_coverage(vocab,embeddings_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('.', 1848),\n",
       " ('the', 1414),\n",
       " ('in', 487),\n",
       " ('is', 398),\n",
       " ('you', 370),\n",
       " ('that', 358),\n",
       " ('i', 337),\n",
       " ('it', 314),\n",
       " ('for', 286),\n",
       " ('with', 271),\n",
       " ('we', 218),\n",
       " ('on', 213),\n",
       " ('as', 212),\n",
       " ('this', 196),\n",
       " ('be', 196),\n",
       " ('are', 188),\n",
       " ('can', 164),\n",
       " ('your', 158),\n",
       " ('have', 155),\n",
       " ('my', 148),\n",
       " (')', 145),\n",
       " ('(', 145),\n",
       " ('but', 140),\n",
       " ('at', 136),\n",
       " ('or', 135),\n",
       " ('from', 126),\n",
       " ('will', 120),\n",
       " ('all', 120),\n",
       " ('our', 120),\n",
       " ('was', 116),\n",
       " ('not', 116),\n",
       " ('its', 107),\n",
       " ('they', 102),\n",
       " ('an', 100),\n",
       " ('so', 98),\n",
       " ('he', 97),\n",
       " ('if', 91),\n",
       " ('what', 90),\n",
       " ('his', 87),\n",
       " ('by', 87),\n",
       " ('more', 86),\n",
       " ('time', 83),\n",
       " ('their', 82),\n",
       " ('would', 80),\n",
       " ('—', 79),\n",
       " ('about', 79),\n",
       " ('how', 78),\n",
       " ('when', 75),\n",
       " ('out', 74),\n",
       " ('do', 72)]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oov[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train.to_csv('cleaned_train_punc_brute.csv',index=False)\n",
    "test.to_csv('cleaned_test_punc_brute.csv',index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}