snli.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from sklearn.model_selection import train_test_split # function for splitting data to train and test sets\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.classify import SklearnClassifier\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import sklearn\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from scipy import sparse\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "snli_train = pd.read_csv('data/snli-train.csv')\n",
    "snli_train = snli_train[['sentence1','sentence2','label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "snli_dev = pd.read_csv('data/snli-train.csv')\n",
    "snli_dev = snli_dev[['sentence1','sentence2','label']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 48: expected 2 fields, saw 3\\nSkipping line 109: expected 2 fields, saw 3\\nSkipping line 126: expected 2 fields, saw 3\\nSkipping line 203: expected 2 fields, saw 3\\nSkipping line 373: expected 2 fields, saw 3\\nSkipping line 570: expected 2 fields, saw 3\\nSkipping line 646: expected 2 fields, saw 3\\nSkipping line 720: expected 2 fields, saw 3\\nSkipping line 744: expected 2 fields, saw 3\\nSkipping line 786: expected 2 fields, saw 3\\nSkipping line 871: expected 2 fields, saw 3\\nSkipping line 885: expected 2 fields, saw 3\\nSkipping line 1058: expected 2 fields, saw 3\\nSkipping line 1081: expected 2 fields, saw 3\\nSkipping line 1104: expected 2 fields, saw 3\\nSkipping line 1138: expected 2 fields, saw 3\\nSkipping line 1239: expected 2 fields, saw 3\\nSkipping line 1254: expected 2 fields, saw 3\\nSkipping line 1281: expected 2 fields, saw 3\\nSkipping line 1348: expected 2 fields, saw 3\\nSkipping line 1354: expected 2 fields, saw 3\\nSkipping line 1361: expected 2 fields, saw 3\\nSkipping line 1368: expected 2 fields, saw 3\\nSkipping line 1460: expected 2 fields, saw 3\\nSkipping line 1461: expected 2 fields, saw 3\\nSkipping line 1486: expected 2 fields, saw 3\\nSkipping line 1488: expected 2 fields, saw 3\\nSkipping line 1495: expected 2 fields, saw 3\\nSkipping line 1537: expected 2 fields, saw 3\\nSkipping line 1555: expected 2 fields, saw 3\\nSkipping line 1565: expected 2 fields, saw 3\\nSkipping line 1815: expected 2 fields, saw 3\\nSkipping line 1882: expected 2 fields, saw 3\\nSkipping line 1908: expected 2 fields, saw 3\\nSkipping line 2018: expected 2 fields, saw 3\\nSkipping line 2069: expected 2 fields, saw 3\\nSkipping line 2116: expected 2 fields, saw 3\\nSkipping line 2123: expected 2 fields, saw 3\\nSkipping line 2152: expected 2 fields, saw 3\\nSkipping line 2157: expected 2 fields, saw 3\\nSkipping line 2203: expected 2 fields, saw 3\\nSkipping line 2352: expected 2 fields, saw 3\\nSkipping line 2356: expected 2 fields, saw 3\\nSkipping line 2368: expected 2 fields, saw 3\\nSkipping line 2389: expected 2 fields, saw 3\\nSkipping line 2398: expected 2 fields, saw 3\\nSkipping line 2405: expected 2 fields, saw 3\\nSkipping line 2407: expected 2 fields, saw 3\\nSkipping line 2425: expected 2 fields, saw 3\\nSkipping line 2510: expected 2 fields, saw 3\\nSkipping line 2637: expected 2 fields, saw 3\\nSkipping line 2661: expected 2 fields, saw 3\\nSkipping line 2826: expected 2 fields, saw 3\\nSkipping line 2844: expected 2 fields, saw 3\\nSkipping line 2875: expected 2 fields, saw 3\\nSkipping line 2934: expected 2 fields, saw 3\\nSkipping line 2949: expected 2 fields, saw 3\\nSkipping line 2962: expected 2 fields, saw 3\\nSkipping line 3117: expected 2 fields, saw 3\\nSkipping line 3154: expected 2 fields, saw 3\\nSkipping line 3166: expected 2 fields, saw 3\\nSkipping line 3190: expected 2 fields, saw 3\\nSkipping line 3215: expected 2 fields, saw 3\\nSkipping line 3412: expected 2 fields, saw 3\\nSkipping line 3418: expected 2 fields, saw 3\\nSkipping line 3505: expected 2 fields, saw 3\\nSkipping line 3558: expected 2 fields, saw 3\\nSkipping line 3640: expected 2 fields, saw 3\\nSkipping line 3646: expected 2 fields, saw 3\\nSkipping line 3756: expected 2 fields, saw 3\\nSkipping line 3788: expected 2 fields, saw 3\\nSkipping line 3900: expected 2 fields, saw 3\\nSkipping line 3963: expected 2 fields, saw 3\\nSkipping line 4090: expected 2 fields, saw 3\\nSkipping line 4124: expected 2 fields, saw 3\\nSkipping line 4130: expected 2 fields, saw 3\\nSkipping line 4192: expected 2 fields, saw 3\\nSkipping line 4220: expected 2 fields, saw 3\\nSkipping line 4227: expected 2 fields, saw 3\\nSkipping line 4307: expected 2 fields, saw 3\\nSkipping line 4329: expected 2 fields, saw 3\\nSkipping line 4352: expected 2 fields, saw 3\\nSkipping line 4444: expected 2 fields, saw 3\\nSkipping line 4682: expected 2 fields, saw 3\\nSkipping line 4769: expected 2 fields, saw 3\\nSkipping line 4774: expected 2 fields, saw 3\\nSkipping line 4828: expected 2 fields, saw 3\\nSkipping line 4993: expected 2 fields, saw 3\\nSkipping line 5010: expected 2 fields, saw 3\\nSkipping line 5176: expected 2 fields, saw 3\\nSkipping line 5383: expected 2 fields, saw 3\\nSkipping line 5452: expected 2 fields, saw 3\\nSkipping line 5489: expected 2 fields, saw 3\\nSkipping line 5494: expected 2 fields, saw 3\\nSkipping line 5506: expected 2 fields, saw 3\\nSkipping line 5582: expected 2 fields, saw 3\\nSkipping line 5779: expected 2 fields, saw 3\\nSkipping line 5804: expected 2 fields, saw 3\\nSkipping line 5812: expected 2 fields, saw 3\\nSkipping line 5879: expected 2 fields, saw 3\\nSkipping line 5920: expected 2 fields, saw 3\\nSkipping line 5933: expected 2 fields, saw 3\\nSkipping line 6005: expected 2 fields, saw 3\\nSkipping line 6042: expected 2 fields, saw 3\\nSkipping line 6046: expected 2 fields, saw 3\\nSkipping line 6110: expected 2 fields, saw 3\\nSkipping line 6131: expected 2 fields, saw 3\\nSkipping line 6136: expected 2 fields, saw 3\\nSkipping line 6216: expected 2 fields, saw 3\\nSkipping line 6232: expected 2 fields, saw 3\\nSkipping line 6243: expected 2 fields, saw 3\\nSkipping line 6422: expected 2 fields, saw 3\\nSkipping line 6423: expected 2 fields, saw 3\\nSkipping line 6508: expected 2 fields, saw 3\\nSkipping line 6621: expected 2 fields, saw 3\\nSkipping line 6644: expected 2 fields, saw 3\\nSkipping line 6653: expected 2 fields, saw 3\\nSkipping line 6722: expected 2 fields, saw 3\\nSkipping line 7060: expected 2 fields, saw 3\\nSkipping line 7082: expected 2 fields, saw 3\\nSkipping line 7109: expected 2 fields, saw 3\\nSkipping line 7134: expected 2 fields, saw 3\\nSkipping line 7138: expected 2 fields, saw 3\\nSkipping line 7143: expected 2 fields, saw 3\\nSkipping line 7215: expected 2 fields, saw 3\\nSkipping line 7222: expected 2 fields, saw 3\\nSkipping line 7238: expected 2 fields, saw 3\\nSkipping line 7262: expected 2 fields, saw 3\\nSkipping line 7291: expected 2 fields, saw 3\\nSkipping line 7320: expected 2 fields, saw 3\\nSkipping line 7469: expected 2 fields, saw 3\\nSkipping line 7483: expected 2 fields, saw 3\\nSkipping line 7488: expected 2 fields, saw 3\\nSkipping line 7578: expected 2 fields, saw 3\\nSkipping line 7630: expected 2 fields, saw 3\\nSkipping line 7670: expected 2 fields, saw 3\\nSkipping line 7742: expected 2 fields, saw 3\\nSkipping line 7828: expected 2 fields, saw 3\\nSkipping line 7832: expected 2 fields, saw 3\\nSkipping line 7867: expected 2 fields, saw 3\\nSkipping line 7922: expected 2 fields, saw 3\\nSkipping line 7985: expected 2 fields, saw 3\\nSkipping line 8030: expected 2 fields, saw 3\\nSkipping line 8045: expected 2 fields, saw 3\\nSkipping line 8071: expected 2 fields, saw 3\\nSkipping line 8136: expected 2 fields, saw 3\\nSkipping line 8156: expected 2 fields, saw 3\\nSkipping line 8234: expected 2 fields, saw 3\\nSkipping line 8385: expected 2 fields, saw 3\\nSkipping line 8395: expected 2 fields, saw 3\\nSkipping line 8454: expected 2 fields, saw 3\\nSkipping line 8510: expected 2 fields, saw 3\\nSkipping line 8571: expected 2 fields, saw 3\\nSkipping line 8601: expected 2 fields, saw 3\\nSkipping line 8616: expected 2 fields, saw 3\\nSkipping line 8643: expected 2 fields, saw 3\\nSkipping line 8867: expected 2 fields, saw 3\\nSkipping line 8888: expected 2 fields, saw 3\\nSkipping line 8949: expected 2 fields, saw 3\\nSkipping line 9032: expected 2 fields, saw 3\\nSkipping line 9050: expected 2 fields, saw 3\\nSkipping line 9053: expected 2 fields, saw 3\\nSkipping line 9067: expected 2 fields, saw 3\\nSkipping line 9126: expected 2 fields, saw 3\\nSkipping line 9273: expected 2 fields, saw 3\\nSkipping line 9275: expected 2 fields, saw 3\\nSkipping line 9304: expected 2 fields, saw 3\\nSkipping line 9309: expected 2 fields, saw 3\\nSkipping line 9514: expected 2 fields, saw 3\\nSkipping line 9515: expected 2 fields, saw 3\\nSkipping line 9570: expected 2 fields, saw 3\\nSkipping line 9591: expected 2 fields, saw 3\\nSkipping line 9613: expected 2 fields, saw 3\\nSkipping line 9692: expected 2 fields, saw 3\\nSkipping line 9725: expected 2 fields, saw 3\\nSkipping line 9957: expected 2 fields, saw 3\\n'\n"
     ]
    }
   ],
   "source": [
    "snli_test = pd.read_csv('data/snli-test.csv',error_bad_lines=False)\n",
    "snli_test = snli_test[['sentence1','sentence2']]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from nltk.stem.wordnet import WordNetLemmatizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#preprocessing Train data\n",
    "import re\n",
    "ps = PorterStemmer()\n",
    "stops = set(stopwords.words(\"english\"))\n",
    "\n",
    "clean_train_sentence1=[]\n",
    "clean_train_sentence2=[]\n",
    "\n",
    "for review in snli_train[\"sentence1\"]:\n",
    "    review = re.sub(r\"[^a-zA-Z]\",\" \", review)\n",
    "    words = review.lower().split()\n",
    "    words = [w for w in words if not w in stops]\n",
    "    #words = [ps.stem(w) for w in words ]\n",
    "    clean_train_sentence1.append(\" \".join(words))\n",
    "\n",
    "    \n",
    "for review in snli_train[\"sentence2\"]:\n",
    "    review = re.sub(r\"[^a-zA-Z]\",\" \", str(review))\n",
    "    words = review.lower().split()\n",
    "    words = [w for w in words if not w in stops]\n",
    "    #words = [ps.stem(w) for w in words ]\n",
    "    clean_train_sentence2.append(\" \".join(words))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "550152"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(snli_train[\"sentence1\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#preprocessing test data\n",
    "\n",
    "clean_test_sentence1=[]\n",
    "clean_test_sentence2=[]\n",
    "\n",
    "for review in snli_test[\"sentence1\"]:\n",
    "    review = re.sub(\"[^a-zA-Z]\",\" \", review)\n",
    "    words = review.lower().split()\n",
    "    words = [w for w in words if not w in stops]\n",
    "    #words = [ps.stem(w) for w in words ]\n",
    "    clean_test_sentence1.append(\" \".join(words))\n",
    "\n",
    "    \n",
    "\n",
    "for review in snli_test[\"sentence2\"]:\n",
    "    review = re.sub(\"[^a-zA-Z]\",\" \", review)\n",
    "    words = review.lower().split()\n",
    "    words = [w for w in words if not w in stops]\n",
    "    #words = [ps.stem(w) for w in words ]\n",
    "    clean_test_sentence2.append(\" \".join(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#preprocessing dev data\n",
    "\n",
    "clean_dev_sentence1=[]\n",
    "clean_dev_sentence2=[]\n",
    "\n",
    "for review in snli_dev[\"sentence1\"]:\n",
    "    review = re.sub(\"[^a-zA-Z]\",\" \", review)\n",
    "    words = review.lower().split()\n",
    "    words = [w for w in words if not w in stops]\n",
    "    #words = [ps.stem(w) for w in words ]\n",
    "    clean_dev_sentence1.append(\" \".join(words))\n",
    "\n",
    "    \n",
    "for review in snli_dev[\"sentence2\"]:\n",
    "    review = re.sub(\"[^a-zA-Z]\",\" \", str(review))\n",
    "    words = review.lower().split()\n",
    "    words = [w for w in words if not w in stops]\n",
    "    #words = [ps.stem(w) for w in words ]\n",
    "    clean_dev_sentence2.append(\" \".join(words))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sklearn\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "count_vec1 = TfidfVectorizer(stop_words='english')\n",
    "count_vec2 = TfidfVectorizer(stop_words='english')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_count2=count_vec2.fit_transform(clean_train_sentence2)\n",
    "train_count1=count_vec1.fit_transform(clean_train_sentence1)\n",
    "train_count=sparse.hstack((train_count1,train_count2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<550152x29480 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 2050499 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_count2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<550152x17488 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 3659784 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_count1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_count2=count_vec2.transform(clean_test_sentence2)\n",
    "test_count1=count_vec1.transform(clean_test_sentence1)\n",
    "test_count=sparse.hstack((test_count1,test_count2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_count2=count_vec2.transform(clean_dev_sentence2)\n",
    "dev_count1=count_vec1.transform(clean_dev_sentence2)\n",
    "dev_count=sparse.hstack((dev_count1,dev_count2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sachinmittal/anaconda3/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5893443981875593"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.cross_validation import KFold,cross_val_score\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "clf = MultinomialNB()\n",
    "scores=cross_val_score(clf,train_count,snli_train[\"label\"],cv=5,scoring='accuracy')\n",
    "scores.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "clf1 = MultinomialNB().fit(train_count,snli_train[\"label\"])\n",
    "y_pred = clf1.predict(test_count)\n",
    "y_pred = pd.DataFrame(y_pred,columns=['label'])\n",
    "frames=[snli_test,y_pred]\n",
    "result=pd.concat(frames,axis=1)\n",
    "result.to_csv(\"snli-test.csv\", sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred_dev = clf1.predict(dev_count)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}